You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/10/24 14:54:35 UTC

[01/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Repository: hbase-site
Updated Branches:
  refs/heads/asf-site 02f92ca7d -> 8f09a71db


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
index 3ec2d53..9400177 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
@@ -33,390 +33,433 @@
 <span class="sourceLineNo">025</span>import java.util.HashMap;<a name="line.25"></a>
 <span class="sourceLineNo">026</span>import java.util.List;<a name="line.26"></a>
 <span class="sourceLineNo">027</span>import java.util.Map;<a name="line.27"></a>
-<span class="sourceLineNo">028</span><a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.hadoop.conf.Configuration;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import org.apache.hadoop.fs.FileStatus;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.apache.hadoop.fs.FileSystem;<a name="line.31"></a>
-<span class="sourceLineNo">032</span>import org.apache.hadoop.fs.FileUtil;<a name="line.32"></a>
-<span class="sourceLineNo">033</span>import org.apache.hadoop.fs.Path;<a name="line.33"></a>
-<span class="sourceLineNo">034</span>import org.apache.hadoop.fs.permission.FsPermission;<a name="line.34"></a>
-<span class="sourceLineNo">035</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.35"></a>
-<span class="sourceLineNo">036</span>import org.apache.hadoop.hbase.HConstants;<a name="line.36"></a>
-<span class="sourceLineNo">037</span>import org.apache.hadoop.hbase.TableName;<a name="line.37"></a>
-<span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.38"></a>
-<span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.ipc.RpcServer;<a name="line.39"></a>
-<span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;<a name="line.40"></a>
-<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.security.User;<a name="line.41"></a>
-<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.42"></a>
-<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.security.token.FsDelegationToken;<a name="line.43"></a>
-<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.security.token.TokenUtil;<a name="line.44"></a>
-<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.45"></a>
-<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.util.FSHDFSUtils;<a name="line.46"></a>
-<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.47"></a>
-<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.util.Methods;<a name="line.48"></a>
-<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.49"></a>
-<span class="sourceLineNo">050</span>import org.apache.hadoop.io.Text;<a name="line.50"></a>
-<span class="sourceLineNo">051</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>import org.apache.hadoop.security.token.Token;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>import org.slf4j.Logger;<a name="line.54"></a>
-<span class="sourceLineNo">055</span>import org.slf4j.LoggerFactory;<a name="line.55"></a>
-<span class="sourceLineNo">056</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;<a name="line.56"></a>
-<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;<a name="line.57"></a>
-<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;<a name="line.58"></a>
-<span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;<a name="line.59"></a>
-<span class="sourceLineNo">060</span><a name="line.60"></a>
-<span class="sourceLineNo">061</span>/**<a name="line.61"></a>
-<span class="sourceLineNo">062</span> * Bulk loads in secure mode.<a name="line.62"></a>
-<span class="sourceLineNo">063</span> *<a name="line.63"></a>
-<span class="sourceLineNo">064</span> * This service addresses two issues:<a name="line.64"></a>
-<span class="sourceLineNo">065</span> * &lt;ol&gt;<a name="line.65"></a>
-<span class="sourceLineNo">066</span> * &lt;li&gt;Moving files in a secure filesystem wherein the HBase Client<a name="line.66"></a>
-<span class="sourceLineNo">067</span> * and HBase Server are different filesystem users.&lt;/li&gt;<a name="line.67"></a>
-<span class="sourceLineNo">068</span> * &lt;li&gt;Does moving in a secure manner. Assuming that the filesystem<a name="line.68"></a>
-<span class="sourceLineNo">069</span> * is POSIX compliant.&lt;/li&gt;<a name="line.69"></a>
-<span class="sourceLineNo">070</span> * &lt;/ol&gt;<a name="line.70"></a>
-<span class="sourceLineNo">071</span> *<a name="line.71"></a>
-<span class="sourceLineNo">072</span> * The algorithm is as follows:<a name="line.72"></a>
-<span class="sourceLineNo">073</span> * &lt;ol&gt;<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * &lt;li&gt;Create an hbase owned staging directory which is<a name="line.74"></a>
-<span class="sourceLineNo">075</span> * world traversable (711): {@code /hbase/staging}&lt;/li&gt;<a name="line.75"></a>
-<span class="sourceLineNo">076</span> * &lt;li&gt;A user writes out data to his secure output directory: {@code /user/foo/data}&lt;/li&gt;<a name="line.76"></a>
-<span class="sourceLineNo">077</span> * &lt;li&gt;A call is made to hbase to create a secret staging directory<a name="line.77"></a>
-<span class="sourceLineNo">078</span> * which globally rwx (777): {@code /user/staging/averylongandrandomdirectoryname}&lt;/li&gt;<a name="line.78"></a>
-<span class="sourceLineNo">079</span> * &lt;li&gt;The user moves the data into the random staging directory,<a name="line.79"></a>
-<span class="sourceLineNo">080</span> * then calls bulkLoadHFiles()&lt;/li&gt;<a name="line.80"></a>
-<span class="sourceLineNo">081</span> * &lt;/ol&gt;<a name="line.81"></a>
-<span class="sourceLineNo">082</span> *<a name="line.82"></a>
-<span class="sourceLineNo">083</span> * Like delegation tokens the strength of the security lies in the length<a name="line.83"></a>
-<span class="sourceLineNo">084</span> * and randomness of the secret directory.<a name="line.84"></a>
-<span class="sourceLineNo">085</span> *<a name="line.85"></a>
-<span class="sourceLineNo">086</span> */<a name="line.86"></a>
-<span class="sourceLineNo">087</span>@InterfaceAudience.Private<a name="line.87"></a>
-<span class="sourceLineNo">088</span>public class SecureBulkLoadManager {<a name="line.88"></a>
-<span class="sourceLineNo">089</span><a name="line.89"></a>
-<span class="sourceLineNo">090</span>  public static final long VERSION = 0L;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span>  //320/5 = 64 characters<a name="line.92"></a>
-<span class="sourceLineNo">093</span>  private static final int RANDOM_WIDTH = 320;<a name="line.93"></a>
-<span class="sourceLineNo">094</span>  private static final int RANDOM_RADIX = 32;<a name="line.94"></a>
+<span class="sourceLineNo">028</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import java.util.function.BiFunction;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import java.util.function.Consumer;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.hadoop.conf.Configuration;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import org.apache.hadoop.fs.FileStatus;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import org.apache.hadoop.fs.FileSystem;<a name="line.34"></a>
+<span class="sourceLineNo">035</span>import org.apache.hadoop.fs.FileUtil;<a name="line.35"></a>
+<span class="sourceLineNo">036</span>import org.apache.hadoop.fs.Path;<a name="line.36"></a>
+<span class="sourceLineNo">037</span>import org.apache.hadoop.fs.permission.FsPermission;<a name="line.37"></a>
+<span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.38"></a>
+<span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.HConstants;<a name="line.39"></a>
+<span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.TableName;<a name="line.40"></a>
+<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.41"></a>
+<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.ipc.RpcServer;<a name="line.42"></a>
+<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;<a name="line.43"></a>
+<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.security.User;<a name="line.44"></a>
+<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.45"></a>
+<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.security.token.FsDelegationToken;<a name="line.46"></a>
+<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.security.token.TokenUtil;<a name="line.47"></a>
+<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.48"></a>
+<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.util.FSHDFSUtils;<a name="line.49"></a>
+<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.50"></a>
+<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.Methods;<a name="line.51"></a>
+<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.52"></a>
+<span class="sourceLineNo">053</span>import org.apache.hadoop.io.Text;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.54"></a>
+<span class="sourceLineNo">055</span>import org.apache.hadoop.security.token.Token;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>import org.slf4j.Logger;<a name="line.57"></a>
+<span class="sourceLineNo">058</span>import org.slf4j.LoggerFactory;<a name="line.58"></a>
+<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.59"></a>
+<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;<a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;<a name="line.63"></a>
+<span class="sourceLineNo">064</span><a name="line.64"></a>
+<span class="sourceLineNo">065</span>/**<a name="line.65"></a>
+<span class="sourceLineNo">066</span> * Bulk loads in secure mode.<a name="line.66"></a>
+<span class="sourceLineNo">067</span> *<a name="line.67"></a>
+<span class="sourceLineNo">068</span> * This service addresses two issues:<a name="line.68"></a>
+<span class="sourceLineNo">069</span> * &lt;ol&gt;<a name="line.69"></a>
+<span class="sourceLineNo">070</span> * &lt;li&gt;Moving files in a secure filesystem wherein the HBase Client<a name="line.70"></a>
+<span class="sourceLineNo">071</span> * and HBase Server are different filesystem users.&lt;/li&gt;<a name="line.71"></a>
+<span class="sourceLineNo">072</span> * &lt;li&gt;Does moving in a secure manner. Assuming that the filesystem<a name="line.72"></a>
+<span class="sourceLineNo">073</span> * is POSIX compliant.&lt;/li&gt;<a name="line.73"></a>
+<span class="sourceLineNo">074</span> * &lt;/ol&gt;<a name="line.74"></a>
+<span class="sourceLineNo">075</span> *<a name="line.75"></a>
+<span class="sourceLineNo">076</span> * The algorithm is as follows:<a name="line.76"></a>
+<span class="sourceLineNo">077</span> * &lt;ol&gt;<a name="line.77"></a>
+<span class="sourceLineNo">078</span> * &lt;li&gt;Create an hbase owned staging directory which is<a name="line.78"></a>
+<span class="sourceLineNo">079</span> * world traversable (711): {@code /hbase/staging}&lt;/li&gt;<a name="line.79"></a>
+<span class="sourceLineNo">080</span> * &lt;li&gt;A user writes out data to his secure output directory: {@code /user/foo/data}&lt;/li&gt;<a name="line.80"></a>
+<span class="sourceLineNo">081</span> * &lt;li&gt;A call is made to hbase to create a secret staging directory<a name="line.81"></a>
+<span class="sourceLineNo">082</span> * which globally rwx (777): {@code /user/staging/averylongandrandomdirectoryname}&lt;/li&gt;<a name="line.82"></a>
+<span class="sourceLineNo">083</span> * &lt;li&gt;The user moves the data into the random staging directory,<a name="line.83"></a>
+<span class="sourceLineNo">084</span> * then calls bulkLoadHFiles()&lt;/li&gt;<a name="line.84"></a>
+<span class="sourceLineNo">085</span> * &lt;/ol&gt;<a name="line.85"></a>
+<span class="sourceLineNo">086</span> *<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * Like delegation tokens the strength of the security lies in the length<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * and randomness of the secret directory.<a name="line.88"></a>
+<span class="sourceLineNo">089</span> *<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span>@InterfaceAudience.Private<a name="line.91"></a>
+<span class="sourceLineNo">092</span>public class SecureBulkLoadManager {<a name="line.92"></a>
+<span class="sourceLineNo">093</span><a name="line.93"></a>
+<span class="sourceLineNo">094</span>  public static final long VERSION = 0L;<a name="line.94"></a>
 <span class="sourceLineNo">095</span><a name="line.95"></a>
-<span class="sourceLineNo">096</span>  private static final Logger LOG = LoggerFactory.getLogger(SecureBulkLoadManager.class);<a name="line.96"></a>
-<span class="sourceLineNo">097</span><a name="line.97"></a>
-<span class="sourceLineNo">098</span>  private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");<a name="line.98"></a>
-<span class="sourceLineNo">099</span>  private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");<a name="line.99"></a>
-<span class="sourceLineNo">100</span>  private SecureRandom random;<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  private FileSystem fs;<a name="line.101"></a>
-<span class="sourceLineNo">102</span>  private Configuration conf;<a name="line.102"></a>
-<span class="sourceLineNo">103</span><a name="line.103"></a>
-<span class="sourceLineNo">104</span>  //two levels so it doesn't get deleted accidentally<a name="line.104"></a>
-<span class="sourceLineNo">105</span>  //no sticky bit in Hadoop 1.0<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  private Path baseStagingDir;<a name="line.106"></a>
+<span class="sourceLineNo">096</span>  //320/5 = 64 characters<a name="line.96"></a>
+<span class="sourceLineNo">097</span>  private static final int RANDOM_WIDTH = 320;<a name="line.97"></a>
+<span class="sourceLineNo">098</span>  private static final int RANDOM_RADIX = 32;<a name="line.98"></a>
+<span class="sourceLineNo">099</span><a name="line.99"></a>
+<span class="sourceLineNo">100</span>  private static final Logger LOG = LoggerFactory.getLogger(SecureBulkLoadManager.class);<a name="line.100"></a>
+<span class="sourceLineNo">101</span><a name="line.101"></a>
+<span class="sourceLineNo">102</span>  private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");<a name="line.102"></a>
+<span class="sourceLineNo">103</span>  private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");<a name="line.103"></a>
+<span class="sourceLineNo">104</span>  private SecureRandom random;<a name="line.104"></a>
+<span class="sourceLineNo">105</span>  private FileSystem fs;<a name="line.105"></a>
+<span class="sourceLineNo">106</span>  private Configuration conf;<a name="line.106"></a>
 <span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  private UserProvider userProvider;<a name="line.108"></a>
-<span class="sourceLineNo">109</span>  private Connection conn;<a name="line.109"></a>
-<span class="sourceLineNo">110</span><a name="line.110"></a>
-<span class="sourceLineNo">111</span>  SecureBulkLoadManager(Configuration conf, Connection conn) {<a name="line.111"></a>
-<span class="sourceLineNo">112</span>    this.conf = conf;<a name="line.112"></a>
-<span class="sourceLineNo">113</span>    this.conn = conn;<a name="line.113"></a>
-<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
+<span class="sourceLineNo">108</span>  //two levels so it doesn't get deleted accidentally<a name="line.108"></a>
+<span class="sourceLineNo">109</span>  //no sticky bit in Hadoop 1.0<a name="line.109"></a>
+<span class="sourceLineNo">110</span>  private Path baseStagingDir;<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span>  private UserProvider userProvider;<a name="line.112"></a>
+<span class="sourceLineNo">113</span>  private ConcurrentHashMap&lt;UserGroupInformation, Integer&gt; ugiReferenceCounter;<a name="line.113"></a>
+<span class="sourceLineNo">114</span>  private Connection conn;<a name="line.114"></a>
 <span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span>  public void start() throws IOException {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>    random = new SecureRandom();<a name="line.117"></a>
-<span class="sourceLineNo">118</span>    userProvider = UserProvider.instantiate(conf);<a name="line.118"></a>
-<span class="sourceLineNo">119</span>    fs = FileSystem.get(conf);<a name="line.119"></a>
-<span class="sourceLineNo">120</span>    baseStagingDir = new Path(FSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);<a name="line.120"></a>
-<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span>    if (conf.get("hbase.bulkload.staging.dir") != null) {<a name="line.122"></a>
-<span class="sourceLineNo">123</span>      LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is "<a name="line.123"></a>
-<span class="sourceLineNo">124</span>          + baseStagingDir);<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    }<a name="line.125"></a>
-<span class="sourceLineNo">126</span>    if (!fs.exists(baseStagingDir)) {<a name="line.126"></a>
-<span class="sourceLineNo">127</span>      fs.mkdirs(baseStagingDir, PERM_HIDDEN);<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    }<a name="line.128"></a>
-<span class="sourceLineNo">129</span>  }<a name="line.129"></a>
-<span class="sourceLineNo">130</span><a name="line.130"></a>
-<span class="sourceLineNo">131</span>  public void stop() throws IOException {<a name="line.131"></a>
-<span class="sourceLineNo">132</span>  }<a name="line.132"></a>
-<span class="sourceLineNo">133</span><a name="line.133"></a>
-<span class="sourceLineNo">134</span>  public String prepareBulkLoad(final HRegion region, final PrepareBulkLoadRequest request)<a name="line.134"></a>
-<span class="sourceLineNo">135</span>      throws IOException {<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    User user = getActiveUser();<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    region.getCoprocessorHost().prePrepareBulkLoad(user);<a name="line.137"></a>
-<span class="sourceLineNo">138</span><a name="line.138"></a>
-<span class="sourceLineNo">139</span>    String bulkToken =<a name="line.139"></a>
-<span class="sourceLineNo">140</span>        createStagingDir(baseStagingDir, user, region.getTableDescriptor().getTableName())<a name="line.140"></a>
-<span class="sourceLineNo">141</span>            .toString();<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span>    return bulkToken;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>  }<a name="line.144"></a>
-<span class="sourceLineNo">145</span><a name="line.145"></a>
-<span class="sourceLineNo">146</span>  public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request)<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throws IOException {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    try {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser());<a name="line.149"></a>
-<span class="sourceLineNo">150</span><a name="line.150"></a>
-<span class="sourceLineNo">151</span>      Path path = new Path(request.getBulkToken());<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      if (!fs.delete(path, true)) {<a name="line.152"></a>
-<span class="sourceLineNo">153</span>        if (fs.exists(path)) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>          throw new IOException("Failed to clean up " + path);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>        }<a name="line.155"></a>
-<span class="sourceLineNo">156</span>      }<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      LOG.info("Cleaned up " + path + " successfully.");<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    } finally {<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      UserGroupInformation ugi = getActiveUser().getUGI();<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      try {<a name="line.160"></a>
-<span class="sourceLineNo">161</span>        if (!UserGroupInformation.getLoginUser().equals(ugi)) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span>          FileSystem.closeAllForUGI(ugi);<a name="line.162"></a>
-<span class="sourceLineNo">163</span>        }<a name="line.163"></a>
-<span class="sourceLineNo">164</span>      } catch (IOException e) {<a name="line.164"></a>
-<span class="sourceLineNo">165</span>        LOG.error("Failed to close FileSystem for: " + ugi, e);<a name="line.165"></a>
-<span class="sourceLineNo">166</span>      }<a name="line.166"></a>
-<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
-<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span>  public Map&lt;byte[], List&lt;Path&gt;&gt; secureBulkLoadHFiles(final HRegion region,<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      final BulkLoadHFileRequest request) throws IOException {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    final List&lt;Pair&lt;byte[], String&gt;&gt; familyPaths = new ArrayList&lt;&gt;(request.getFamilyPathCount());<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {<a name="line.173"></a>
-<span class="sourceLineNo">174</span>      familyPaths.add(new Pair&lt;&gt;(el.getFamily().toByteArray(), el.getPath()));<a name="line.174"></a>
-<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
-<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span>    Token userToken = null;<a name="line.177"></a>
-<span class="sourceLineNo">178</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span>      userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()<a name="line.179"></a>
-<span class="sourceLineNo">180</span>              .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(<a name="line.180"></a>
-<span class="sourceLineNo">181</span>              request.getFsToken().getService()));<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    }<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    final String bulkToken = request.getBulkToken();<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    User user = getActiveUser();<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    final UserGroupInformation ugi = user.getUGI();<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      try {<a name="line.187"></a>
-<span class="sourceLineNo">188</span>        Token tok = TokenUtil.obtainToken(conn);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>        if (tok != null) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span>          boolean b = ugi.addToken(tok);<a name="line.190"></a>
-<span class="sourceLineNo">191</span>          LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>        }<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      } catch (IOException ioe) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span>        LOG.warn("unable to add token", ioe);<a name="line.194"></a>
-<span class="sourceLineNo">195</span>      }<a name="line.195"></a>
-<span class="sourceLineNo">196</span>    }<a name="line.196"></a>
-<span class="sourceLineNo">197</span>    if (userToken != null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      ugi.addToken(userToken);<a name="line.198"></a>
-<span class="sourceLineNo">199</span>    } else if (userProvider.isHadoopSecurityEnabled()) {<a name="line.199"></a>
-<span class="sourceLineNo">200</span>      //we allow this to pass through in "simple" security mode<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      //for mini cluster testing<a name="line.201"></a>
-<span class="sourceLineNo">202</span>      throw new DoNotRetryIOException("User token cannot be null");<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span>    if (region.getCoprocessorHost() != null) {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>      region.getCoprocessorHost().preBulkLoadHFile(familyPaths);<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    }<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    Map&lt;byte[], List&lt;Path&gt;&gt; map = null;<a name="line.208"></a>
-<span class="sourceLineNo">209</span><a name="line.209"></a>
-<span class="sourceLineNo">210</span>    try {<a name="line.210"></a>
-<span class="sourceLineNo">211</span>      // Get the target fs (HBase region server fs) delegation token<a name="line.211"></a>
-<span class="sourceLineNo">212</span>      // Since we have checked the permission via 'preBulkLoadHFile', now let's give<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      // the 'request user' necessary token to operate on the target fs.<a name="line.213"></a>
-<span class="sourceLineNo">214</span>      // After this point the 'doAs' user will hold two tokens, one for the source fs<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      // ('request user'), another for the target fs (HBase region server principal).<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      if (userProvider.isHadoopSecurityEnabled()) {<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider,"renewer");<a name="line.217"></a>
-<span class="sourceLineNo">218</span>        targetfsDelegationToken.acquireDelegationToken(fs);<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>        Token&lt;?&gt; targetFsToken = targetfsDelegationToken.getUserToken();<a name="line.220"></a>
-<span class="sourceLineNo">221</span>        if (targetFsToken != null<a name="line.221"></a>
-<span class="sourceLineNo">222</span>            &amp;&amp; (userToken == null || !targetFsToken.getService().equals(userToken.getService()))){<a name="line.222"></a>
-<span class="sourceLineNo">223</span>          ugi.addToken(targetFsToken);<a name="line.223"></a>
-<span class="sourceLineNo">224</span>        }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>      }<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>      map = ugi.doAs(new PrivilegedAction&lt;Map&lt;byte[], List&lt;Path&gt;&gt;&gt;() {<a name="line.227"></a>
-<span class="sourceLineNo">228</span>        @Override<a name="line.228"></a>
-<span class="sourceLineNo">229</span>        public Map&lt;byte[], List&lt;Path&gt;&gt; run() {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>          FileSystem fs = null;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>          try {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>            fs = FileSystem.get(conf);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>            for(Pair&lt;byte[], String&gt; el: familyPaths) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span>              Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));<a name="line.234"></a>
-<span class="sourceLineNo">235</span>              if(!fs.exists(stageFamily)) {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>                fs.mkdirs(stageFamily);<a name="line.236"></a>
-<span class="sourceLineNo">237</span>                fs.setPermission(stageFamily, PERM_ALL_ACCESS);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>              }<a name="line.238"></a>
-<span class="sourceLineNo">239</span>            }<a name="line.239"></a>
-<span class="sourceLineNo">240</span>            //We call bulkLoadHFiles as requesting user<a name="line.240"></a>
-<span class="sourceLineNo">241</span>            //To enable access prior to staging<a name="line.241"></a>
-<span class="sourceLineNo">242</span>            return region.bulkLoadHFiles(familyPaths, true,<a name="line.242"></a>
-<span class="sourceLineNo">243</span>                new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile());<a name="line.243"></a>
-<span class="sourceLineNo">244</span>          } catch (Exception e) {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>            LOG.error("Failed to complete bulk load", e);<a name="line.245"></a>
-<span class="sourceLineNo">246</span>          }<a name="line.246"></a>
-<span class="sourceLineNo">247</span>          return null;<a name="line.247"></a>
-<span class="sourceLineNo">248</span>        }<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      });<a name="line.249"></a>
-<span class="sourceLineNo">250</span>    } finally {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      if (region.getCoprocessorHost() != null) {<a name="line.251"></a>
-<span class="sourceLineNo">252</span>        region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map);<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    }<a name="line.254"></a>
-<span class="sourceLineNo">255</span>    return map;<a name="line.255"></a>
-<span class="sourceLineNo">256</span>  }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span>  private Path createStagingDir(Path baseDir,<a name="line.258"></a>
-<span class="sourceLineNo">259</span>                                User user,<a name="line.259"></a>
-<span class="sourceLineNo">260</span>                                TableName tableName) throws IOException {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>    String tblName = tableName.getNameAsString().replace(":", "_");<a name="line.261"></a>
-<span class="sourceLineNo">262</span>    String randomDir = user.getShortName()+"__"+ tblName +"__"+<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));<a name="line.263"></a>
-<span class="sourceLineNo">264</span>    return createStagingDir(baseDir, user, randomDir);<a name="line.264"></a>
-<span class="sourceLineNo">265</span>  }<a name="line.265"></a>
-<span class="sourceLineNo">266</span><a name="line.266"></a>
-<span class="sourceLineNo">267</span>  private Path createStagingDir(Path baseDir,<a name="line.267"></a>
-<span class="sourceLineNo">268</span>                                User user,<a name="line.268"></a>
-<span class="sourceLineNo">269</span>                                String randomDir) throws IOException {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    Path p = new Path(baseDir, randomDir);<a name="line.270"></a>
-<span class="sourceLineNo">271</span>    fs.mkdirs(p, PERM_ALL_ACCESS);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>    fs.setPermission(p, PERM_ALL_ACCESS);<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    return p;<a name="line.273"></a>
-<span class="sourceLineNo">274</span>  }<a name="line.274"></a>
-<span class="sourceLineNo">275</span><a name="line.275"></a>
-<span class="sourceLineNo">276</span>  private User getActiveUser() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>    // for non-rpc handling, fallback to system user<a name="line.277"></a>
-<span class="sourceLineNo">278</span>    User user = RpcServer.getRequestUser().orElse(userProvider.getCurrent());<a name="line.278"></a>
-<span class="sourceLineNo">279</span>    // this is for testing<a name="line.279"></a>
-<span class="sourceLineNo">280</span>    if (userProvider.isHadoopSecurityEnabled() &amp;&amp;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>        "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {<a name="line.281"></a>
-<span class="sourceLineNo">282</span>      return User.createUserForTesting(conf, user.getShortName(), new String[] {});<a name="line.282"></a>
-<span class="sourceLineNo">283</span>    }<a name="line.283"></a>
-<span class="sourceLineNo">284</span><a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return user;<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  private static class SecureBulkLoadListener implements BulkLoadListener {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    // Target filesystem<a name="line.289"></a>
-<span class="sourceLineNo">290</span>    private final FileSystem fs;<a name="line.290"></a>
-<span class="sourceLineNo">291</span>    private final String stagingDir;<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    private final Configuration conf;<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    // Source filesystem<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    private FileSystem srcFs = null;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>    private Map&lt;String, FsPermission&gt; origPermissions = null;<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>    public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {<a name="line.297"></a>
-<span class="sourceLineNo">298</span>      this.fs = fs;<a name="line.298"></a>
-<span class="sourceLineNo">299</span>      this.stagingDir = stagingDir;<a name="line.299"></a>
-<span class="sourceLineNo">300</span>      this.conf = conf;<a name="line.300"></a>
-<span class="sourceLineNo">301</span>      this.origPermissions = new HashMap&lt;&gt;();<a name="line.301"></a>
-<span class="sourceLineNo">302</span>    }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span>    @Override<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile)<a name="line.305"></a>
-<span class="sourceLineNo">306</span>        throws IOException {<a name="line.306"></a>
-<span class="sourceLineNo">307</span>      Path p = new Path(srcPath);<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.308"></a>
+<span class="sourceLineNo">116</span>  SecureBulkLoadManager(Configuration conf, Connection conn) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span>    this.conf = conf;<a name="line.117"></a>
+<span class="sourceLineNo">118</span>    this.conn = conn;<a name="line.118"></a>
+<span class="sourceLineNo">119</span>  }<a name="line.119"></a>
+<span class="sourceLineNo">120</span><a name="line.120"></a>
+<span class="sourceLineNo">121</span>  public void start() throws IOException {<a name="line.121"></a>
+<span class="sourceLineNo">122</span>    random = new SecureRandom();<a name="line.122"></a>
+<span class="sourceLineNo">123</span>    userProvider = UserProvider.instantiate(conf);<a name="line.123"></a>
+<span class="sourceLineNo">124</span>    ugiReferenceCounter = new ConcurrentHashMap&lt;&gt;();<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    fs = FileSystem.get(conf);<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    baseStagingDir = new Path(FSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>    if (conf.get("hbase.bulkload.staging.dir") != null) {<a name="line.128"></a>
+<span class="sourceLineNo">129</span>      LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is "<a name="line.129"></a>
+<span class="sourceLineNo">130</span>          + baseStagingDir);<a name="line.130"></a>
+<span class="sourceLineNo">131</span>    }<a name="line.131"></a>
+<span class="sourceLineNo">132</span>    if (!fs.exists(baseStagingDir)) {<a name="line.132"></a>
+<span class="sourceLineNo">133</span>      fs.mkdirs(baseStagingDir, PERM_HIDDEN);<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    }<a name="line.134"></a>
+<span class="sourceLineNo">135</span>  }<a name="line.135"></a>
+<span class="sourceLineNo">136</span><a name="line.136"></a>
+<span class="sourceLineNo">137</span>  public void stop() throws IOException {<a name="line.137"></a>
+<span class="sourceLineNo">138</span>  }<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span>  public String prepareBulkLoad(final HRegion region, final PrepareBulkLoadRequest request)<a name="line.140"></a>
+<span class="sourceLineNo">141</span>      throws IOException {<a name="line.141"></a>
+<span class="sourceLineNo">142</span>    User user = getActiveUser();<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    region.getCoprocessorHost().prePrepareBulkLoad(user);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span>    String bulkToken =<a name="line.145"></a>
+<span class="sourceLineNo">146</span>        createStagingDir(baseStagingDir, user, region.getTableDescriptor().getTableName())<a name="line.146"></a>
+<span class="sourceLineNo">147</span>            .toString();<a name="line.147"></a>
+<span class="sourceLineNo">148</span><a name="line.148"></a>
+<span class="sourceLineNo">149</span>    return bulkToken;<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request)<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    try {<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser());<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span>      Path path = new Path(request.getBulkToken());<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      if (!fs.delete(path, true)) {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>        if (fs.exists(path)) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>          throw new IOException("Failed to clean up " + path);<a name="line.160"></a>
+<span class="sourceLineNo">161</span>        }<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      }<a name="line.162"></a>
+<span class="sourceLineNo">163</span>      LOG.info("Cleaned up " + path + " successfully.");<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    } finally {<a name="line.164"></a>
+<span class="sourceLineNo">165</span>      UserGroupInformation ugi = getActiveUser().getUGI();<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      try {<a name="line.166"></a>
+<span class="sourceLineNo">167</span>        if (!UserGroupInformation.getLoginUser().equals(ugi) &amp;&amp; !isUserReferenced(ugi)) {<a name="line.167"></a>
+<span class="sourceLineNo">168</span>          FileSystem.closeAllForUGI(ugi);<a name="line.168"></a>
+<span class="sourceLineNo">169</span>        }<a name="line.169"></a>
+<span class="sourceLineNo">170</span>      } catch (IOException e) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>        LOG.error("Failed to close FileSystem for: " + ugi, e);<a name="line.171"></a>
+<span class="sourceLineNo">172</span>      }<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  }<a name="line.174"></a>
+<span class="sourceLineNo">175</span><a name="line.175"></a>
+<span class="sourceLineNo">176</span>  private Consumer&lt;HRegion&gt; fsCreatedListener;<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  @VisibleForTesting<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  void setFsCreatedListener(Consumer&lt;HRegion&gt; fsCreatedListener) {<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    this.fsCreatedListener = fsCreatedListener;<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  }<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span>  private void incrementUgiReference(UserGroupInformation ugi) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>    ugiReferenceCounter.merge(ugi, 1, new BiFunction&lt;Integer, Integer, Integer&gt;() {<a name="line.185"></a>
+<span class="sourceLineNo">186</span>      @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span>      public Integer apply(Integer oldvalue, Integer value) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span>        return ++oldvalue;<a name="line.188"></a>
+<span class="sourceLineNo">189</span>      }<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    });<a name="line.190"></a>
+<span class="sourceLineNo">191</span>  }<a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span>  private void decrementUgiReference(UserGroupInformation ugi) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    ugiReferenceCounter.computeIfPresent(ugi,<a name="line.194"></a>
+<span class="sourceLineNo">195</span>        new BiFunction&lt;UserGroupInformation, Integer, Integer&gt;() {<a name="line.195"></a>
+<span class="sourceLineNo">196</span>          @Override<a name="line.196"></a>
+<span class="sourceLineNo">197</span>          public Integer apply(UserGroupInformation key, Integer value) {<a name="line.197"></a>
+<span class="sourceLineNo">198</span>            return value &gt; 1 ? --value : null;<a name="line.198"></a>
+<span class="sourceLineNo">199</span>          }<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      });<a name="line.200"></a>
+<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
+<span class="sourceLineNo">202</span><a name="line.202"></a>
+<span class="sourceLineNo">203</span>  private boolean isUserReferenced(UserGroupInformation ugi) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>    Integer count = ugiReferenceCounter.get(ugi);<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    return count != null &amp;&amp; count &gt; 0;<a name="line.205"></a>
+<span class="sourceLineNo">206</span>  }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>  public Map&lt;byte[], List&lt;Path&gt;&gt; secureBulkLoadHFiles(final HRegion region,<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      final BulkLoadHFileRequest request) throws IOException {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    final List&lt;Pair&lt;byte[], String&gt;&gt; familyPaths = new ArrayList&lt;&gt;(request.getFamilyPathCount());<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>      familyPaths.add(new Pair&lt;&gt;(el.getFamily().toByteArray(), el.getPath()));<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    }<a name="line.213"></a>
+<span class="sourceLineNo">214</span><a name="line.214"></a>
+<span class="sourceLineNo">215</span>    Token userToken = null;<a name="line.215"></a>
+<span class="sourceLineNo">216</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.216"></a>
+<span class="sourceLineNo">217</span>      userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()<a name="line.217"></a>
+<span class="sourceLineNo">218</span>              .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(<a name="line.218"></a>
+<span class="sourceLineNo">219</span>              request.getFsToken().getService()));<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    final String bulkToken = request.getBulkToken();<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    User user = getActiveUser();<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    final UserGroupInformation ugi = user.getUGI();<a name="line.223"></a>
+<span class="sourceLineNo">224</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      try {<a name="line.225"></a>
+<span class="sourceLineNo">226</span>        Token tok = TokenUtil.obtainToken(conn);<a name="line.226"></a>
+<span class="sourceLineNo">227</span>        if (tok != null) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span>          boolean b = ugi.addToken(tok);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>          LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);<a name="line.229"></a>
+<span class="sourceLineNo">230</span>        }<a name="line.230"></a>
+<span class="sourceLineNo">231</span>      } catch (IOException ioe) {<a name="line.231"></a>
+<span class="sourceLineNo">232</span>        LOG.warn("unable to add token", ioe);<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      }<a name="line.233"></a>
+<span class="sourceLineNo">234</span>    }<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    if (userToken != null) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span>      ugi.addToken(userToken);<a name="line.236"></a>
+<span class="sourceLineNo">237</span>    } else if (userProvider.isHadoopSecurityEnabled()) {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>      //we allow this to pass through in "simple" security mode<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      //for mini cluster testing<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      throw new DoNotRetryIOException("User token cannot be null");<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    }<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span>    if (region.getCoprocessorHost() != null) {<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      region.getCoprocessorHost().preBulkLoadHFile(familyPaths);<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    Map&lt;byte[], List&lt;Path&gt;&gt; map = null;<a name="line.246"></a>
+<span class="sourceLineNo">247</span><a name="line.247"></a>
+<span class="sourceLineNo">248</span>    try {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      incrementUgiReference(ugi);<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      // Get the target fs (HBase region server fs) delegation token<a name="line.250"></a>
+<span class="sourceLineNo">251</span>      // Since we have checked the permission via 'preBulkLoadHFile', now let's give<a name="line.251"></a>
+<span class="sourceLineNo">252</span>      // the 'request user' necessary token to operate on the target fs.<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      // After this point the 'doAs' user will hold two tokens, one for the source fs<a name="line.253"></a>
+<span class="sourceLineNo">254</span>      // ('request user'), another for the target fs (HBase region server principal).<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      if (userProvider.isHadoopSecurityEnabled()) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider,"renewer");<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        targetfsDelegationToken.acquireDelegationToken(fs);<a name="line.257"></a>
+<span class="sourceLineNo">258</span><a name="line.258"></a>
+<span class="sourceLineNo">259</span>        Token&lt;?&gt; targetFsToken = targetfsDelegationToken.getUserToken();<a name="line.259"></a>
+<span class="sourceLineNo">260</span>        if (targetFsToken != null<a name="line.260"></a>
+<span class="sourceLineNo">261</span>            &amp;&amp; (userToken == null || !targetFsToken.getService().equals(userToken.getService()))){<a name="line.261"></a>
+<span class="sourceLineNo">262</span>          ugi.addToken(targetFsToken);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>        }<a name="line.263"></a>
+<span class="sourceLineNo">264</span>      }<a name="line.264"></a>
+<span class="sourceLineNo">265</span><a name="line.265"></a>
+<span class="sourceLineNo">266</span>      map = ugi.doAs(new PrivilegedAction&lt;Map&lt;byte[], List&lt;Path&gt;&gt;&gt;() {<a name="line.266"></a>
+<span class="sourceLineNo">267</span>        @Override<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        public Map&lt;byte[], List&lt;Path&gt;&gt; run() {<a name="line.268"></a>
+<span class="sourceLineNo">269</span>          FileSystem fs = null;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>          try {<a name="line.270"></a>
+<span class="sourceLineNo">271</span>            fs = FileSystem.get(conf);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>            for(Pair&lt;byte[], String&gt; el: familyPaths) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span>              Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));<a name="line.273"></a>
+<span class="sourceLineNo">274</span>              if(!fs.exists(stageFamily)) {<a name="line.274"></a>
+<span class="sourceLineNo">275</span>                fs.mkdirs(stageFamily);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>                fs.setPermission(stageFamily, PERM_ALL_ACCESS);<a name="line.276"></a>
+<span class="sourceLineNo">277</span>              }<a name="line.277"></a>
+<span class="sourceLineNo">278</span>            }<a name="line.278"></a>
+<span class="sourceLineNo">279</span>            if (fsCreatedListener != null) {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>              fsCreatedListener.accept(region);<a name="line.280"></a>
+<span class="sourceLineNo">281</span>            }<a name="line.281"></a>
+<span class="sourceLineNo">282</span>            //We call bulkLoadHFiles as requesting user<a name="line.282"></a>
+<span class="sourceLineNo">283</span>            //To enable access prior to staging<a name="line.283"></a>
+<span class="sourceLineNo">284</span>            return region.bulkLoadHFiles(familyPaths, true,<a name="line.284"></a>
+<span class="sourceLineNo">285</span>                new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile());<a name="line.285"></a>
+<span class="sourceLineNo">286</span>          } catch (Exception e) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>            LOG.error("Failed to complete bulk load", e);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>          }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>          return null;<a name="line.289"></a>
+<span class="sourceLineNo">290</span>        }<a name="line.290"></a>
+<span class="sourceLineNo">291</span>      });<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    } finally {<a name="line.292"></a>
+<span class="sourceLineNo">293</span>      decrementUgiReference(ugi);<a name="line.293"></a>
+<span class="sourceLineNo">294</span>      if (region.getCoprocessorHost() != null) {<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map);<a name="line.295"></a>
+<span class="sourceLineNo">296</span>      }<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    }<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return map;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  private Path createStagingDir(Path baseDir,<a name="line.301"></a>
+<span class="sourceLineNo">302</span>                                User user,<a name="line.302"></a>
+<span class="sourceLineNo">303</span>                                TableName tableName) throws IOException {<a name="line.303"></a>
+<span class="sourceLineNo">304</span>    String tblName = tableName.getNameAsString().replace(":", "_");<a name="line.304"></a>
+<span class="sourceLineNo">305</span>    String randomDir = user.getShortName()+"__"+ tblName +"__"+<a name="line.305"></a>
+<span class="sourceLineNo">306</span>        (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));<a name="line.306"></a>
+<span class="sourceLineNo">307</span>    return createStagingDir(baseDir, user, randomDir);<a name="line.307"></a>
+<span class="sourceLineNo">308</span>  }<a name="line.308"></a>
 <span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span>      // In case of Replication for bulk load files, hfiles are already copied in staging directory<a name="line.310"></a>
-<span class="sourceLineNo">311</span>      if (p.equals(stageP)) {<a name="line.311"></a>
-<span class="sourceLineNo">312</span>        LOG.debug(p.getName()<a name="line.312"></a>
-<span class="sourceLineNo">313</span>            + " is already available in staging directory. Skipping copy or rename.");<a name="line.313"></a>
-<span class="sourceLineNo">314</span>        return stageP.toString();<a name="line.314"></a>
-<span class="sourceLineNo">315</span>      }<a name="line.315"></a>
-<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span>      if (srcFs == null) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.318"></a>
-<span class="sourceLineNo">319</span>      }<a name="line.319"></a>
-<span class="sourceLineNo">320</span><a name="line.320"></a>
-<span class="sourceLineNo">321</span>      if(!isFile(p)) {<a name="line.321"></a>
-<span class="sourceLineNo">322</span>        throw new IOException("Path does not reference a file: " + p);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>      }<a name="line.323"></a>
-<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span>      // Check to see if the source and target filesystems are the same<a name="line.325"></a>
-<span class="sourceLineNo">326</span>      if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span>        LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            "the destination filesystem. Copying file over to destination staging dir.");<a name="line.328"></a>
-<span class="sourceLineNo">329</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>      } else if (copyFile) {<a name="line.330"></a>
-<span class="sourceLineNo">331</span>        LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");<a name="line.331"></a>
-<span class="sourceLineNo">332</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.332"></a>
-<span class="sourceLineNo">333</span>      } else {<a name="line.333"></a>
-<span class="sourceLineNo">334</span>        LOG.debug("Moving " + p + " to " + stageP);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>        FileStatus origFileStatus = fs.getFileStatus(p);<a name="line.335"></a>
-<span class="sourceLineNo">336</span>        origPermissions.put(srcPath, origFileStatus.getPermission());<a name="line.336"></a>
-<span class="sourceLineNo">337</span>        if(!fs.rename(p, stageP)) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          throw new IOException("Failed to move HFile: " + p + " to " + stageP);<a name="line.338"></a>
-<span class="sourceLineNo">339</span>        }<a name="line.339"></a>
-<span class="sourceLineNo">340</span>      }<a name="line.340"></a>
-<span class="sourceLineNo">341</span>      fs.setPermission(stageP, PERM_ALL_ACCESS);<a name="line.341"></a>
-<span class="sourceLineNo">342</span>      return stageP.toString();<a name="line.342"></a>
-<span class="sourceLineNo">343</span>    }<a name="line.343"></a>
-<span class="sourceLineNo">344</span><a name="line.344"></a>
-<span class="sourceLineNo">345</span>    @Override<a name="line.345"></a>
-<span class="sourceLineNo">346</span>    public void doneBulkLoad(byte[] family, String srcPath) throws IOException {<a name="line.346"></a>
-<span class="sourceLineNo">347</span>      LOG.debug("Bulk Load done for: " + srcPath);<a name="line.347"></a>
-<span class="sourceLineNo">348</span>      closeSrcFs();<a name="line.348"></a>
-<span class="sourceLineNo">349</span>    }<a name="line.349"></a>
-<span class="sourceLineNo">350</span><a name="line.350"></a>
-<span class="sourceLineNo">351</span>    private void closeSrcFs() throws IOException {<a name="line.351"></a>
-<span class="sourceLineNo">352</span>      if (srcFs != null) {<a name="line.352"></a>
-<span class="sourceLineNo">353</span>        srcFs.close();<a name="line.353"></a>
-<span class="sourceLineNo">354</span>        srcFs = null;<a name="line.354"></a>
-<span class="sourceLineNo">355</span>      }<a name="line.355"></a>
-<span class="sourceLineNo">356</span>    }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>    @Override<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {<a name="line.359"></a>
-<span class="sourceLineNo">360</span>      try {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>        Path p = new Path(srcPath);<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        if (srcFs == null) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span>          srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.363"></a>
-<span class="sourceLineNo">364</span>        }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>        if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.365"></a>
-<span class="sourceLineNo">366</span>          // files are copied so no need to move them back<a name="line.366"></a>
-<span class="sourceLineNo">367</span>          return;<a name="line.367"></a>
-<span class="sourceLineNo">368</span>        }<a name="line.368"></a>
-<span class="sourceLineNo">369</span>        Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span>        // In case of Replication for bulk load files, hfiles are not renamed by end point during<a name="line.371"></a>
-<span class="sourceLineNo">372</span>        // prepare stage, so no need of rename here again<a name="line.372"></a>
-<span class="sourceLineNo">373</span>        if (p.equals(stageP)) {<a name="line.373"></a>
-<span class="sourceLineNo">374</span>          LOG.debug(p.getName() + " is already available in source directory. Skipping rename.");<a name="line.374"></a>
-<span class="sourceLineNo">375</span>          return;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>        }<a name="line.376"></a>
-<span class="sourceLineNo">377</span><a name="line.377"></a>
-<span class="sourceLineNo">378</span>        LOG.debug("Moving " + stageP + " back to " + p);<a name="line.378"></a>
-<span class="sourceLineNo">379</span>        if (!fs.rename(stageP, p)) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span>          throw new IOException("Failed to move HFile: " + stageP + " to " + p);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>        }<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>        // restore original permission<a name="line.383"></a>
-<span class="sourceLineNo">384</span>        if (origPermissions.containsKey(srcPath)) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>          fs.setPermission(p, origPermissions.get(srcPath));<a name="line.385"></a>
-<span class="sourceLineNo">386</span>        } else {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>          LOG.warn("Can't find previous permission for path=" + srcPath);<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        }<a name="line.388"></a>
-<span class="sourceLineNo">389</span>      } finally {<a name="line.389"></a>
-<span class="sourceLineNo">390</span>        closeSrcFs();<a name="line.390"></a>
-<span class="sourceLineNo">391</span>      }<a name="line.391"></a>
+<span class="sourceLineNo">310</span>  private Path createStagingDir(Path baseDir,<a name="line.310"></a>
+<span class="sourceLineNo">311</span>                                User user,<a name="line.311"></a>
+<span class="sourceLineNo">312</span>                                String randomDir) throws IOException {<a name="line.312"></a>
+<span class="sourceLineNo">313</span>    Path p = new Path(baseDir, randomDir);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>    fs.mkdirs(p, PERM_ALL_ACCESS);<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    fs.setPermission(p, PERM_ALL_ACCESS);<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return p;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private User getActiveUser() throws IOException {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    // for non-rpc handling, fallback to system user<a name="line.320"></a>
+<span class="sourceLineNo">321</span>    User user = RpcServer.getRequestUser().orElse(userProvider.getCurrent());<a name="line.321"></a>
+<span class="sourceLineNo">322</span>    // this is for testing<a name="line.322"></a>
+<span class="sourceLineNo">323</span>    if (userProvider.isHadoopSecurityEnabled() &amp;&amp;<a name="line.323"></a>
+<span class="sourceLineNo">324</span>        "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {<a name="line.324"></a>
+<span class="sourceLineNo">325</span>      return User.createUserForTesting(conf, user.getShortName(), new String[] {});<a name="line.325"></a>
+<span class="sourceLineNo">326</span>    }<a name="line.326"></a>
+<span class="sourceLineNo">327</span><a name="line.327"></a>
+<span class="sourceLineNo">328</span>    return user;<a name="line.328"></a>
+<span class="sourceLineNo">329</span>  }<a name="line.329"></a>
+<span class="sourceLineNo">330</span><a name="line.330"></a>
+<span class="sourceLineNo">331</span>  private static class SecureBulkLoadListener implements BulkLoadListener {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>    // Target filesystem<a name="line.332"></a>
+<span class="sourceLineNo">333</span>    private final FileSystem fs;<a name="line.333"></a>
+<span class="sourceLineNo">334</span>    private final String stagingDir;<a name="line.334"></a>
+<span class="sourceLineNo">335</span>    private final Configuration conf;<a name="line.335"></a>
+<span class="sourceLineNo">336</span>    // Source filesystem<a name="line.336"></a>
+<span class="sourceLineNo">337</span>    private FileSystem srcFs = null;<a name="line.337"></a>
+<span class="sourceLineNo">338</span>    private Map&lt;String, FsPermission&gt; origPermissions = null;<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span>    public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {<a name="line.340"></a>
+<span class="sourceLineNo">341</span>      this.fs = fs;<a name="line.341"></a>
+<span class="sourceLineNo">342</span>      this.stagingDir = stagingDir;<a name="line.342"></a>
+<span class="sourceLineNo">343</span>      this.conf = conf;<a name="line.343"></a>
+<span class="sourceLineNo">344</span>      this.origPermissions = new HashMap&lt;&gt;();<a name="line.344"></a>
+<span class="sourceLineNo">345</span>    }<a name="line.345"></a>
+<span class="sourceLineNo">346</span><a name="line.346"></a>
+<span class="sourceLineNo">347</span>    @Override<a name="line.347"></a>
+<span class="sourceLineNo">348</span>    public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile)<a name="line.348"></a>
+<span class="sourceLineNo">349</span>        throws IOException {<a name="line.349"></a>
+<span class="sourceLineNo">350</span>      Path p = new Path(srcPath);<a name="line.350"></a>
+<span class="sourceLineNo">351</span>      Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.351"></a>
+<span class="sourceLineNo">352</span><a name="line.352"></a>
+<span class="sourceLineNo">353</span>      // In case of Replication for bulk load files, hfiles are already copied in staging directory<a name="line.353"></a>
+<span class="sourceLineNo">354</span>      if (p.equals(stageP)) {<a name="line.354"></a>
+<span class="sourceLineNo">355</span>        LOG.debug(p.getName()<a name="line.355"></a>
+<span class="sourceLineNo">356</span>            + " is already available in staging directory. Skipping copy or rename.");<a name="line.356"></a>
+<span class="sourceLineNo">357</span>        return stageP.toString();<a name="line.357"></a>
+<span class="sourceLineNo">358</span>      }<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span>      if (srcFs == null) {<a name="line.360"></a>
+<span class="sourceLineNo">361</span>        srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.361"></a>
+<span class="sourceLineNo">362</span>      }<a name="line.362"></a>
+<span class="sourceLineNo">363</span><a name="line.363"></a>
+<span class="sourceLineNo">364</span>      if(!isFile(p)) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>        throw new IOException("Path does not reference a file: " + p);<a name="line.365"></a>
+<span class="sourceLineNo">366</span>      }<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span>      // Check to see if the source and target filesystems are the same<a name="line.368"></a>
+<span class="sourceLineNo">369</span>      if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.369"></a>
+<span class="sourceLineNo">370</span>        LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +<a name="line.370"></a>
+<span class="sourceLineNo">371</span>            "the destination filesystem. Copying file over to destination staging dir.");<a name="line.371"></a>
+<span class="sourceLineNo">372</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.372"></a>
+<span class="sourceLineNo">373</span>      } else if (copyFile) {<a name="line.373"></a>
+<span class="sourceLineNo">374</span>        LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");<a name="line.374"></a>
+<span class="sourceLineNo">375</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.375"></a>
+<span class="sourceLineNo">376</span>      } else {<a name="line.376"></a>
+<span class="sourceLineNo">377</span>        LOG.debug("Moving " + p + " to " + stageP);<a name="line.377"></a>
+<span class="sourceLineNo">378</span>        FileStatus origFileStatus = fs.getFileStatus(p);<a name="line.378"></a>
+<span class="sourceLineNo">379</span>        origPermissions.put(srcPath, origFileStatus.getPermission());<a name="line.379"></a>
+<span class="sourceLineNo">380</span>        if(!fs.rename(p, stageP)) {<a name="line.380"></a>
+<span class="sourceLineNo">381</span>          throw new IOException("Failed to move HFile: " + p + " to " + stageP);<a name="line.381"></a>
+<span class="sourceLineNo">382</span>        }<a name="line.382"></a>
+<span class="sourceLineNo">383</span>      }<a name="line.383"></a>
+<span class="sourceLineNo">384</span>      fs.setPermission(stageP, PERM_ALL_ACCESS);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      return stageP.toString();<a name="line.385"></a>
+<span class="sourceLineNo">386</span>    }<a name="line.386"></a>
+<span class="sourceLineNo">387</span><a name="line.387"></a>
+<span class="sourceLineNo">388</span>    @Override<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    public void doneBulkLoad(byte[] family, String srcPath) throws IOException {<a name="line.389"></a>
+<span class="sourceLineNo">390</span>      LOG.debug("Bulk Load done for: " + srcPath);<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      closeSrcFs();<a name="line.391"></a>
 <span class="sourceLineNo">392</span>    }<a name="line.392"></a>
 <span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    /**<a name="line.394"></a>
-<span class="sourceLineNo">395</span>     * Check if the path is referencing a file.<a name="line.395"></a>
-<span class="sourceLineNo">396</span>     * This is mainly needed to avoid symlinks.<a name="line.396"></a>
-<span class="sourceLineNo">397</span>     * @param p<a name="line.397"></a>
-<span class="sourceLineNo">398</span>     * @return true if the p is a file<a name="line.398"></a>
-<span class="sourceLineNo">399</span>     * @throws IOException<a name="line.399"></a>
-<span class="sourceLineNo">400</span>     */<a name="line.400"></a>
-<span class="sourceLineNo">401</span>    private boolean isFile(Path p) throws IOException {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      FileStatus status = srcFs.getFileStatus(p);<a name="line.402"></a>
-<span class="sourceLineNo">403</span>      boolean isFile = !status.isDirectory();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>      try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>        isFile = isFile &amp;&amp; !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);<a name="line.405"></a>
-<span class="sourceLineNo">406</span>      } catch (Exception e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
-<span class="sourceLineNo">408</span>      return isFile;<a name="line.408"></a>
-<span class="sourceLineNo">409</span>    }<a name="line.409"></a>
-<span class="sourceLineNo">410</span>  }<a name="line.410"></a>
-<span class="sourceLineNo">411</span>}<a name="line.411"></a>
+<span class="sourceLineNo">394</span>    private void closeSrcFs() throws IOException {<a name="line.394"></a>
+<span class="sourceLineNo">395</span>      if (srcFs != null) {<a name="line.395"></a>
+<span class="sourceLineNo">396</span>        srcFs.close();<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        srcFs = null;<a name="line.397"></a>
+<span class="sourceLineNo">398</span>      }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>    }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span>    @Override<a name="line.401"></a>
+<span class="sourceLineNo">402</span>    public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      try {<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        Path p = new Path(srcPath);<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        if (srcFs == null) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.406"></a>
+<span class="sourceLineNo">407</span>        }<a name="line.407"></a>
+<span class="sourceLineNo">408</span>        if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>          // files are copied so no need to move them back<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          return;<a name="line.410"></a>
+<span class="sourceLineNo">411</span>        }<a name="line.411"></a>
+<span class="sourceLineNo">412</span>        Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>        // In case of Replication for bulk load files, hfiles are not renamed by end point during<a name="line.414"></a>
+<span class="sourceLineNo">415</span>        // prepare stage, so no need of rename here again<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        if (p.equals(stageP)) {<a name="line.416"></a>
+<span class="sourceLineNo">417</span>          LOG.debug(p.getName() + " is already available in source directory. Skipping rename.");<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          return;<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        }<a name="line.419"></a>
+<span class="sourceLineNo">420</span><a name="line.420"></a>
+<span class="sourceLineNo">421</span>        LOG.debug("Moving " + stageP + " back to " + p);<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        if (!fs.rename(stageP, p)) {<a name="line.422"></a>
+<span class="sourceLineNo">423</span>          throw new IOException("Failed to move HFile: " + stageP + " to " + p);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        }<a name="line.424"></a>
+<span class="sourceLineNo">425</span><a name="line.425"></a>
+<span class="sourceLineNo">426</span>        // restore original permission<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        if (origPermissions.containsKey(srcPath)) {<a name="line.427"></a>
+<span class="sourceLineNo">428</span>          fs.setPermission(p, origPermissions.get(srcPath));<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        } else {<a name="line.429"></a>
+<span class="sourceLineNo">430</span>          LOG.warn("Can't find previous permission for path=" + srcPath);<a name="line.430"></a>
+<span class="sourceLineNo">431</span>        }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      } finally {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        closeSrcFs();<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>    /**<a name="line.437"></a>
+<span class="sourceLineNo">438</span>     * Check if the path is referencing a file.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>     * This is mainly needed to avoid symlinks.<a name="line.439"></a>
+<span class="sourceLineNo">440</span>     * @param p<a name="line.440"></a>
+<span class="sourceLineNo">441</span>     * @return true if the p is a file<a name="line.441"></a>
+<span class="sourceLineNo">442</span>     * @throws IOException<a name="line.442"></a>
+<span class="sourceLineNo">443</span>     */<a name="line.443"></a>
+<span class="sourceLineNo">444</span>    private boolean isFile(Path p) throws IOException {<a name="line.444"></a>
+<span class="sourceLineNo">445</span>      FileStatus status = srcFs.getFileStatus(p);<a name="line.445"></a>
+<span class="sourceLineNo">446</span>      boolean isFile = !status.isDirectory();<a name="line.446"></a>
+<span class="sourceLineNo">447</span>      try {<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        isFile = isFile &amp;&amp; !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      } catch (Exception e) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>      return isFile;<a name="line.451"></a>
+<span class="sourceLineNo">452</span>    }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>  }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>}<a name="line.454"></a>
 
 
 


[28/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class=

<TRUNCATED>

[33/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<s

<TRUNCATED>

[09/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
index 30963fa..257263c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.InvalidWALDataException.html
@@ -81,187 +81,198 @@
 <span class="sourceLineNo">073</span><a name="line.73"></a>
 <span class="sourceLineNo">074</span>  private ProcedureWALFormat() {}<a name="line.74"></a>
 <span class="sourceLineNo">075</span><a name="line.75"></a>
-<span class="sourceLineNo">076</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.76"></a>
-<span class="sourceLineNo">077</span>      Loader loader) throws IOException {<a name="line.77"></a>
-<span class="sourceLineNo">078</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.78"></a>
-<span class="sourceLineNo">079</span>    tracker.setKeepDeletes(true);<a name="line.79"></a>
-<span class="sourceLineNo">080</span>    try {<a name="line.80"></a>
-<span class="sourceLineNo">081</span>      // Ignore the last log which is current active log.<a name="line.81"></a>
-<span class="sourceLineNo">082</span>      while (logs.hasNext()) {<a name="line.82"></a>
-<span class="sourceLineNo">083</span>        ProcedureWALFile log = logs.next();<a name="line.83"></a>
-<span class="sourceLineNo">084</span>        log.open();<a name="line.84"></a>
-<span class="sourceLineNo">085</span>        try {<a name="line.85"></a>
-<span class="sourceLineNo">086</span>          reader.read(log);<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        } finally {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          log.close();<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        }<a name="line.89"></a>
-<span class="sourceLineNo">090</span>      }<a name="line.90"></a>
-<span class="sourceLineNo">091</span>      reader.finish();<a name="line.91"></a>
-<span class="sourceLineNo">092</span><a name="line.92"></a>
-<span class="sourceLineNo">093</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.93"></a>
-<span class="sourceLineNo">094</span>      if (tracker.isPartial()) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span>        tracker.setPartialFlag(false);<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      }<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetModified();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
+<span class="sourceLineNo">076</span>  /**<a name="line.76"></a>
+<span class="sourceLineNo">077</span>   * Load all the procedures in these ProcedureWALFiles, and rebuild the given {@code tracker} if<a name="line.77"></a>
+<span class="sourceLineNo">078</span>   * needed, i.e, the {@code tracker} is a partial one.<a name="line.78"></a>
+<span class="sourceLineNo">079</span>   * &lt;p/&gt;<a name="line.79"></a>
+<span class="sourceLineNo">080</span>   * The method in the give {@code loader} will be called at the end after we load all the<a name="line.80"></a>
+<span class="sourceLineNo">081</span>   * procedures and construct the hierarchy.<a name="line.81"></a>
+<span class="sourceLineNo">082</span>   * &lt;p/&gt;<a name="line.82"></a>
+<span class="sourceLineNo">083</span>   * And we will call the {@link ProcedureStoreTracker#resetModified()} method for the given<a name="line.83"></a>
+<span class="sourceLineNo">084</span>   * {@code tracker} before returning, as it will be used to track the next proc wal file's modified<a name="line.84"></a>
+<span class="sourceLineNo">085</span>   * procedures.<a name="line.85"></a>
+<span class="sourceLineNo">086</span>   */<a name="line.86"></a>
+<span class="sourceLineNo">087</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.87"></a>
+<span class="sourceLineNo">088</span>      Loader loader) throws IOException {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    tracker.setKeepDeletes(true);<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    try {<a name="line.91"></a>
+<span class="sourceLineNo">092</span>      // Ignore the last log which is current active log.<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      while (logs.hasNext()) {<a name="line.93"></a>
+<span class="sourceLineNo">094</span>        ProcedureWALFile log = logs.next();<a name="line.94"></a>
+<span class="sourceLineNo">095</span>        log.open();<a name="line.95"></a>
+<span class="sourceLineNo">096</span>        try {<a name="line.96"></a>
+<span class="sourceLineNo">097</span>          reader.read(log);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>        } finally {<a name="line.98"></a>
+<span class="sourceLineNo">099</span>          log.close();<a name="line.99"></a>
+<span class="sourceLineNo">100</span>        }<a name="line.100"></a>
+<span class="sourceLineNo">101</span>      }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>      reader.finish();<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      if (tracker.isPartial()) {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>        tracker.setPartialFlag(false);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>      }<a name="line.107"></a>
+<span class="sourceLineNo">108</span>      tracker.resetModified();<a name="line.108"></a>
+<span class="sourceLineNo">109</span>    } finally {<a name="line.109"></a>
+<span class="sourceLineNo">110</span>      tracker.setKeepDeletes(false);<a name="line.110"></a>
+<span class="sourceLineNo">111</span>    }<a name="line.111"></a>
+<span class="sourceLineNo">112</span>  }<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.114"></a>
+<span class="sourceLineNo">115</span>      throws IOException {<a name="line.115"></a>
+<span class="sourceLineNo">116</span>    header.writeDelimitedTo(stream);<a name="line.116"></a>
+<span class="sourceLineNo">117</span>  }<a name="line.117"></a>
+<span class="sourceLineNo">118</span><a name="line.118"></a>
+<span class="sourceLineNo">119</span>  /*<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * | END OF WAL DATA | &lt;---+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+     |<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   * |                 |     |<a name="line.123"></a>
+<span class="sourceLineNo">124</span>   * |     Tracker     |     |<a name="line.124"></a>
+<span class="sourceLineNo">125</span>   * |                 |     |<a name="line.125"></a>
+<span class="sourceLineNo">126</span>   * +-----------------+     |<a name="line.126"></a>
+<span class="sourceLineNo">127</span>   * |     version     |     |<a name="line.127"></a>
+<span class="sourceLineNo">128</span>   * +-----------------+     |<a name="line.128"></a>
+<span class="sourceLineNo">129</span>   * |  TRAILER_MAGIC  |     |<a name="line.129"></a>
+<span class="sourceLineNo">130</span>   * +-----------------+     |<a name="line.130"></a>
+<span class="sourceLineNo">131</span>   * |      offset     |-----+<a name="line.131"></a>
+<span class="sourceLineNo">132</span>   * +-----------------+<a name="line.132"></a>
+<span class="sourceLineNo">133</span>   */<a name="line.133"></a>
+<span class="sourceLineNo">134</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.134"></a>
+<span class="sourceLineNo">135</span>      throws IOException {<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    long offset = stream.getPos();<a name="line.136"></a>
+<span class="sourceLineNo">137</span><a name="line.137"></a>
+<span class="sourceLineNo">138</span>    // Write EOF Entry<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    ProcedureWALEntry.newBuilder()<a name="line.139"></a>
+<span class="sourceLineNo">140</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.140"></a>
+<span class="sourceLineNo">141</span>      .build().writeDelimitedTo(stream);<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span>    // Write Tracker<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.144"></a>
+<span class="sourceLineNo">145</span><a name="line.145"></a>
+<span class="sourceLineNo">146</span>    stream.write(TRAILER_VERSION);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    StreamUtils.writeLong(stream, offset);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    return stream.getPos() - offset;<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    ProcedureWALHeader header;<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    try {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    } catch (InvalidProtocolBufferException e) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      throw new InvalidWALDataException(e);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>    if (header == null) {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.166"></a>
+<span class="sourceLineNo">167</span>          " expected " + HEADER_VERSION);<a name="line.167"></a>
+<span class="sourceLineNo">168</span>    }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
+<span class="sourceLineNo">173</span><a name="line.173"></a>
+<span class="sourceLineNo">174</span>    return header;<a name="line.174"></a>
+<span class="sourceLineNo">175</span>  }<a name="line.175"></a>
+<span class="sourceLineNo">176</span><a name="line.176"></a>
+<span class="sourceLineNo">177</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.177"></a>
+<span class="sourceLineNo">178</span>      throws IOException {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    long trailerPos = size - 17;<a name="line.180"></a>
 <span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
+<span class="sourceLineNo">182</span>    if (trailerPos &lt; startPos) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span>    stream.seek(trailerPos);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    int version = stream.read();<a name="line.187"></a>
+<span class="sourceLineNo">188</span>    if (version != TRAILER_VERSION) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.189"></a>
+<span class="sourceLineNo">190</span>          " expected " + TRAILER_VERSION);<a name="line.190"></a>
+<span class="sourceLineNo">191</span>    }<a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span>    long magic = StreamUtils.readLong(stream);<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    if (magic != TRAILER_MAGIC) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.195"></a>
+<span class="sourceLineNo">196</span>          " expected " + TRAILER_MAGIC);<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    }<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>    stream.seek(trailerOffset);<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    }<a name="line.205"></a>
 <span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">207</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      .setVersion(version)<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      .setTrackerPos(stream.getPos())<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      .build();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    return trailer;<a name="line.211"></a>
+<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>  }<a name="line.216"></a>
+<span class="sourceLineNo">217</span><a name="line.217"></a>
+<span class="sourceLineNo">218</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    builder.setType(type);<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    if (subprocs != null) {<a name="line.223"></a>
+<span class="sourceLineNo">224</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.224"></a>
+<span class="sourceLineNo">225</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.225"></a>
+<span class="sourceLineNo">226</span>      }<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    }<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    builder.build().writeDelimitedTo(slot);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.238"></a>
+<span class="sourceLineNo">239</span>  }<a name="line.239"></a>
+<span class="sourceLineNo">240</span><a name="line.240"></a>
+<span class="sourceLineNo">241</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      throws IOException {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.243"></a>
+<span class="sourceLineNo">244</span>  }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      throws IOException {<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    builder.setProcId(procId);<a name="line.250"></a>
+<span class="sourceLineNo">251</span>    builder.build().writeDelimitedTo(slot);<a name="line.251"></a>
+<span class="sourceLineNo">252</span>  }<a name="line.252"></a>
+<span class="sourceLineNo">253</span><a name="line.253"></a>
+<span class="sourceLineNo">254</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      throws IOException {<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.256"></a>
+<span class="sourceLineNo">257</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.257"></a>
+<span class="sourceLineNo">258</span>    builder.setProcId(proc.getProcId());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    if (subprocs != null) {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span>        builder.addChildId(subprocs[i]);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>      }<a name="line.263"></a>
+<span class="sourceLineNo">264</span>    }<a name="line.264"></a>
+<span class="sourceLineNo">265</span>    builder.build().writeDelimitedTo(slot);<a name="line.265"></a>
+<span class="sourceLineNo">266</span>  }<a name="line.266"></a>
+<span class="sourceLineNo">267</span>}<a name="line.267"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
index 30963fa..257263c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html
@@ -81,187 +81,198 @@
 <span class="sourceLineNo">073</span><a name="line.73"></a>
 <span class="sourceLineNo">074</span>  private ProcedureWALFormat() {}<a name="line.74"></a>
 <span class="sourceLineNo">075</span><a name="line.75"></a>
-<span class="sourceLineNo">076</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.76"></a>
-<span class="sourceLineNo">077</span>      Loader loader) throws IOException {<a name="line.77"></a>
-<span class="sourceLineNo">078</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.78"></a>
-<span class="sourceLineNo">079</span>    tracker.setKeepDeletes(true);<a name="line.79"></a>
-<span class="sourceLineNo">080</span>    try {<a name="line.80"></a>
-<span class="sourceLineNo">081</span>      // Ignore the last log which is current active log.<a name="line.81"></a>
-<span class="sourceLineNo">082</span>      while (logs.hasNext()) {<a name="line.82"></a>
-<span class="sourceLineNo">083</span>        ProcedureWALFile log = logs.next();<a name="line.83"></a>
-<span class="sourceLineNo">084</span>        log.open();<a name="line.84"></a>
-<span class="sourceLineNo">085</span>        try {<a name="line.85"></a>
-<span class="sourceLineNo">086</span>          reader.read(log);<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        } finally {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          log.close();<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        }<a name="line.89"></a>
-<span class="sourceLineNo">090</span>      }<a name="line.90"></a>
-<span class="sourceLineNo">091</span>      reader.finish();<a name="line.91"></a>
-<span class="sourceLineNo">092</span><a name="line.92"></a>
-<span class="sourceLineNo">093</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.93"></a>
-<span class="sourceLineNo">094</span>      if (tracker.isPartial()) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span>        tracker.setPartialFlag(false);<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      }<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetModified();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
+<span class="sourceLineNo">076</span>  /**<a name="line.76"></a>
+<span class="sourceLineNo">077</span>   * Load all the procedures in these ProcedureWALFiles, and rebuild the given {@code tracker} if<a name="line.77"></a>
+<span class="sourceLineNo">078</span>   * needed, i.e, the {@code tracker} is a partial one.<a name="line.78"></a>
+<span class="sourceLineNo">079</span>   * &lt;p/&gt;<a name="line.79"></a>
+<span class="sourceLineNo">080</span>   * The method in the give {@code loader} will be called at the end after we load all the<a name="line.80"></a>
+<span class="sourceLineNo">081</span>   * procedures and construct the hierarchy.<a name="line.81"></a>
+<span class="sourceLineNo">082</span>   * &lt;p/&gt;<a name="line.82"></a>
+<span class="sourceLineNo">083</span>   * And we will call the {@link ProcedureStoreTracker#resetModified()} method for the given<a name="line.83"></a>
+<span class="sourceLineNo">084</span>   * {@code tracker} before returning, as it will be used to track the next proc wal file's modified<a name="line.84"></a>
+<span class="sourceLineNo">085</span>   * procedures.<a name="line.85"></a>
+<span class="sourceLineNo">086</span>   */<a name="line.86"></a>
+<span class="sourceLineNo">087</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.87"></a>
+<span class="sourceLineNo">088</span>      Loader loader) throws IOException {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    tracker.setKeepDeletes(true);<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    try {<a name="line.91"></a>
+<span class="sourceLineNo">092</span>      // Ignore the last log which is current active log.<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      while (logs.hasNext()) {<a name="line.93"></a>
+<span class="sourceLineNo">094</span>        ProcedureWALFile log = logs.next();<a name="line.94"></a>
+<span class="sourceLineNo">095</span>        log.open();<a name="line.95"></a>
+<span class="sourceLineNo">096</span>        try {<a name="line.96"></a>
+<span class="sourceLineNo">097</span>          reader.read(log);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>        } finally {<a name="line.98"></a>
+<span class="sourceLineNo">099</span>          log.close();<a name="line.99"></a>
+<span class="sourceLineNo">100</span>        }<a name="line.100"></a>
+<span class="sourceLineNo">101</span>      }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>      reader.finish();<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      if (tracker.isPartial()) {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>        tracker.setPartialFlag(false);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>      }<a name="line.107"></a>
+<span class="sourceLineNo">108</span>      tracker.resetModified();<a name="line.108"></a>
+<span class="sourceLineNo">109</span>    } finally {<a name="line.109"></a>
+<span class="sourceLineNo">110</span>      tracker.setKeepDeletes(false);<a name="line.110"></a>
+<span class="sourceLineNo">111</span>    }<a name="line.111"></a>
+<span class="sourceLineNo">112</span>  }<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.114"></a>
+<span class="sourceLineNo">115</span>      throws IOException {<a name="line.115"></a>
+<span class="sourceLineNo">116</span>    header.writeDelimitedTo(stream);<a name="line.116"></a>
+<span class="sourceLineNo">117</span>  }<a name="line.117"></a>
+<span class="sourceLineNo">118</span><a name="line.118"></a>
+<span class="sourceLineNo">119</span>  /*<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * | END OF WAL DATA | &lt;---+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+     |<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   * |                 |     |<a name="line.123"></a>
+<span class="sourceLineNo">124</span>   * |     Tracker     |     |<a name="line.124"></a>
+<span class="sourceLineNo">125</span>   * |                 |     |<a name="line.125"></a>
+<span class="sourceLineNo">126</span>   * +-----------------+     |<a name="line.126"></a>
+<span class="sourceLineNo">127</span>   * |     version     |     |<a name="line.127"></a>
+<span class="sourceLineNo">128</span>   * +-----------------+     |<a name="line.128"></a>
+<span class="sourceLineNo">129</span>   * |  TRAILER_MAGIC  |     |<a name="line.129"></a>
+<span class="sourceLineNo">130</span>   * +-----------------+     |<a name="line.130"></a>
+<span class="sourceLineNo">131</span>   * |      offset     |-----+<a name="line.131"></a>
+<span class="sourceLineNo">132</span>   * +-----------------+<a name="line.132"></a>
+<span class="sourceLineNo">133</span>   */<a name="line.133"></a>
+<span class="sourceLineNo">134</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.134"></a>
+<span class="sourceLineNo">135</span>      throws IOException {<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    long offset = stream.getPos();<a name="line.136"></a>
+<span class="sourceLineNo">137</span><a name="line.137"></a>
+<span class="sourceLineNo">138</span>    // Write EOF Entry<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    ProcedureWALEntry.newBuilder()<a name="line.139"></a>
+<span class="sourceLineNo">140</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.140"></a>
+<span class="sourceLineNo">141</span>      .build().writeDelimitedTo(stream);<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span>    // Write Tracker<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.144"></a>
+<span class="sourceLineNo">145</span><a name="line.145"></a>
+<span class="sourceLineNo">146</span>    stream.write(TRAILER_VERSION);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    StreamUtils.writeLong(stream, offset);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    return stream.getPos() - offset;<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    ProcedureWALHeader header;<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    try {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    } catch (InvalidProtocolBufferException e) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      throw new InvalidWALDataException(e);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>    if (header == null) {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.166"></a>
+<span class="sourceLineNo">167</span>          " expected " + HEADER_VERSION);<a name="line.167"></a>
+<span class="sourceLineNo">168</span>    }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
+<span class="sourceLineNo">173</span><a name="line.173"></a>
+<span class="sourceLineNo">174</span>    return header;<a name="line.174"></a>
+<span class="sourceLineNo">175</span>  }<a name="line.175"></a>
+<span class="sourceLineNo">176</span><a name="line.176"></a>
+<span class="sourceLineNo">177</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.177"></a>
+<span class="sourceLineNo">178</span>      throws IOException {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    long trailerPos = size - 17;<a name="line.180"></a>
 <span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
+<span class="sourceLineNo">182</span>    if (trailerPos &lt; startPos) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span>    stream.seek(trailerPos);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    int version = stream.read();<a name="line.187"></a>
+<span class="sourceLineNo">188</span>    if (version != TRAILER_VERSION) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.189"></a>
+<span class="sourceLineNo">190</span>          " expected " + TRAILER_VERSION);<a name="line.190"></a>
+<span class="sourceLineNo">191</span>    }<a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span>    long magic = StreamUtils.readLong(stream);<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    if (magic != TRAILER_MAGIC) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.195"></a>
+<span class="sourceLineNo">196</span>          " expected " + TRAILER_MAGIC);<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    }<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>    stream.seek(trailerOffset);<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    }<a name="line.205"></a>
 <span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">207</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      .setVersion(version)<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      .setTrackerPos(stream.getPos())<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      .build();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    return trailer;<a name="line.211"></a>
+<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>  }<a name="line.216"></a>
+<span class="sourceLineNo">217</span><a name="line.217"></a>
+<span class="sourceLineNo">218</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    builder.setType(type);<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    if (subprocs != null) {<a name="line.223"></a>
+<span class="sourceLineNo">224</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.224"></a>
+<span class="sourceLineNo">225</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.225"></a>
+<span class="sourceLineNo">226</span>      }<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    }<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    builder.build().writeDelimitedTo(slot);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.238"></a>
+<span class="sourceLineNo">239</span>  }<a name="line.239"></a>
+<span class="sourceLineNo">240</span><a name="line.240"></a>
+<span class="sourceLineNo">241</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      throws IOException {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.243"></a>
+<span class="sourceLineNo">244</span>  }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      throws IOException {<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    builder.setProcId(procId);<a name="line.250"></a>
+<span class="sourceLineNo">251</span>    builder.build().writeDelimitedTo(slot);<a name="line.251"></a>
+<span class="sourceLineNo">252</span>  }<a name="line.252"></a>
+<span class="sourceLineNo">253</span><a name="line.253"></a>
+<span class="sourceLineNo">254</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      throws IOException {<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.256"></a>
+<span class="sourceLineNo">257</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.257"></a>
+<span class="sourceLineNo">258</span>    builder.setProcId(proc.getProcId());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    if (subprocs != null) {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span>        builder.addChildId(subprocs[i]);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>      }<a name="line.263"></a>
+<span class="sourceLineNo">264</span>    }<a name="line.264"></a>
+<span class="sourceLineNo">265</span>    builder.build().writeDelimitedTo(slot);<a name="line.265"></a>
+<span class="sourceLineNo">266</span>  }<a name="line.266"></a>
+<span class="sourceLineNo">267</span>}<a name="line.267"></a>
 
 
 


[29/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span c

<TRUNCATED>

[08/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
index 30963fa..257263c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
@@ -81,187 +81,198 @@
 <span class="sourceLineNo">073</span><a name="line.73"></a>
 <span class="sourceLineNo">074</span>  private ProcedureWALFormat() {}<a name="line.74"></a>
 <span class="sourceLineNo">075</span><a name="line.75"></a>
-<span class="sourceLineNo">076</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.76"></a>
-<span class="sourceLineNo">077</span>      Loader loader) throws IOException {<a name="line.77"></a>
-<span class="sourceLineNo">078</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.78"></a>
-<span class="sourceLineNo">079</span>    tracker.setKeepDeletes(true);<a name="line.79"></a>
-<span class="sourceLineNo">080</span>    try {<a name="line.80"></a>
-<span class="sourceLineNo">081</span>      // Ignore the last log which is current active log.<a name="line.81"></a>
-<span class="sourceLineNo">082</span>      while (logs.hasNext()) {<a name="line.82"></a>
-<span class="sourceLineNo">083</span>        ProcedureWALFile log = logs.next();<a name="line.83"></a>
-<span class="sourceLineNo">084</span>        log.open();<a name="line.84"></a>
-<span class="sourceLineNo">085</span>        try {<a name="line.85"></a>
-<span class="sourceLineNo">086</span>          reader.read(log);<a name="line.86"></a>
-<span class="sourceLineNo">087</span>        } finally {<a name="line.87"></a>
-<span class="sourceLineNo">088</span>          log.close();<a name="line.88"></a>
-<span class="sourceLineNo">089</span>        }<a name="line.89"></a>
-<span class="sourceLineNo">090</span>      }<a name="line.90"></a>
-<span class="sourceLineNo">091</span>      reader.finish();<a name="line.91"></a>
-<span class="sourceLineNo">092</span><a name="line.92"></a>
-<span class="sourceLineNo">093</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.93"></a>
-<span class="sourceLineNo">094</span>      if (tracker.isPartial()) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span>        tracker.setPartialFlag(false);<a name="line.95"></a>
-<span class="sourceLineNo">096</span>      }<a name="line.96"></a>
-<span class="sourceLineNo">097</span>      tracker.resetModified();<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    } finally {<a name="line.98"></a>
-<span class="sourceLineNo">099</span>      tracker.setKeepDeletes(false);<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  }<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.103"></a>
-<span class="sourceLineNo">104</span>      throws IOException {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>    header.writeDelimitedTo(stream);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  }<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  /*<a name="line.108"></a>
-<span class="sourceLineNo">109</span>   * +-----------------+<a name="line.109"></a>
-<span class="sourceLineNo">110</span>   * | END OF WAL DATA | &lt;---+<a name="line.110"></a>
-<span class="sourceLineNo">111</span>   * +-----------------+     |<a name="line.111"></a>
-<span class="sourceLineNo">112</span>   * |                 |     |<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * |     Tracker     |     |<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   * |                 |     |<a name="line.114"></a>
-<span class="sourceLineNo">115</span>   * +-----------------+     |<a name="line.115"></a>
-<span class="sourceLineNo">116</span>   * |     version     |     |<a name="line.116"></a>
-<span class="sourceLineNo">117</span>   * +-----------------+     |<a name="line.117"></a>
-<span class="sourceLineNo">118</span>   * |  TRAILER_MAGIC  |     |<a name="line.118"></a>
-<span class="sourceLineNo">119</span>   * +-----------------+     |<a name="line.119"></a>
-<span class="sourceLineNo">120</span>   * |      offset     |-----+<a name="line.120"></a>
-<span class="sourceLineNo">121</span>   * +-----------------+<a name="line.121"></a>
-<span class="sourceLineNo">122</span>   */<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      throws IOException {<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    long offset = stream.getPos();<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span>    // Write EOF Entry<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    ProcedureWALEntry.newBuilder()<a name="line.128"></a>
-<span class="sourceLineNo">129</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.129"></a>
-<span class="sourceLineNo">130</span>      .build().writeDelimitedTo(stream);<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span>    // Write Tracker<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.133"></a>
-<span class="sourceLineNo">134</span><a name="line.134"></a>
-<span class="sourceLineNo">135</span>    stream.write(TRAILER_VERSION);<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    StreamUtils.writeLong(stream, offset);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    return stream.getPos() - offset;<a name="line.138"></a>
-<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
-<span class="sourceLineNo">140</span><a name="line.140"></a>
-<span class="sourceLineNo">141</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    ProcedureWALHeader header;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    try {<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } catch (InvalidProtocolBufferException e) {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throw new InvalidWALDataException(e);<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>    if (header == null) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    }<a name="line.152"></a>
-<span class="sourceLineNo">153</span><a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.155"></a>
-<span class="sourceLineNo">156</span>          " expected " + HEADER_VERSION);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>    return header;<a name="line.163"></a>
-<span class="sourceLineNo">164</span>  }<a name="line.164"></a>
-<span class="sourceLineNo">165</span><a name="line.165"></a>
-<span class="sourceLineNo">166</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.166"></a>
-<span class="sourceLineNo">167</span>      throws IOException {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    long trailerPos = size - 17;<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>    if (trailerPos &lt; startPos) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>    stream.seek(trailerPos);<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    int version = stream.read();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    if (version != TRAILER_VERSION) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.178"></a>
-<span class="sourceLineNo">179</span>          " expected " + TRAILER_VERSION);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    }<a name="line.180"></a>
+<span class="sourceLineNo">076</span>  /**<a name="line.76"></a>
+<span class="sourceLineNo">077</span>   * Load all the procedures in these ProcedureWALFiles, and rebuild the given {@code tracker} if<a name="line.77"></a>
+<span class="sourceLineNo">078</span>   * needed, i.e, the {@code tracker} is a partial one.<a name="line.78"></a>
+<span class="sourceLineNo">079</span>   * &lt;p/&gt;<a name="line.79"></a>
+<span class="sourceLineNo">080</span>   * The method in the give {@code loader} will be called at the end after we load all the<a name="line.80"></a>
+<span class="sourceLineNo">081</span>   * procedures and construct the hierarchy.<a name="line.81"></a>
+<span class="sourceLineNo">082</span>   * &lt;p/&gt;<a name="line.82"></a>
+<span class="sourceLineNo">083</span>   * And we will call the {@link ProcedureStoreTracker#resetModified()} method for the given<a name="line.83"></a>
+<span class="sourceLineNo">084</span>   * {@code tracker} before returning, as it will be used to track the next proc wal file's modified<a name="line.84"></a>
+<span class="sourceLineNo">085</span>   * procedures.<a name="line.85"></a>
+<span class="sourceLineNo">086</span>   */<a name="line.86"></a>
+<span class="sourceLineNo">087</span>  public static void load(Iterator&lt;ProcedureWALFile&gt; logs, ProcedureStoreTracker tracker,<a name="line.87"></a>
+<span class="sourceLineNo">088</span>      Loader loader) throws IOException {<a name="line.88"></a>
+<span class="sourceLineNo">089</span>    ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader);<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    tracker.setKeepDeletes(true);<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    try {<a name="line.91"></a>
+<span class="sourceLineNo">092</span>      // Ignore the last log which is current active log.<a name="line.92"></a>
+<span class="sourceLineNo">093</span>      while (logs.hasNext()) {<a name="line.93"></a>
+<span class="sourceLineNo">094</span>        ProcedureWALFile log = logs.next();<a name="line.94"></a>
+<span class="sourceLineNo">095</span>        log.open();<a name="line.95"></a>
+<span class="sourceLineNo">096</span>        try {<a name="line.96"></a>
+<span class="sourceLineNo">097</span>          reader.read(log);<a name="line.97"></a>
+<span class="sourceLineNo">098</span>        } finally {<a name="line.98"></a>
+<span class="sourceLineNo">099</span>          log.close();<a name="line.99"></a>
+<span class="sourceLineNo">100</span>        }<a name="line.100"></a>
+<span class="sourceLineNo">101</span>      }<a name="line.101"></a>
+<span class="sourceLineNo">102</span>      reader.finish();<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span>      // The tracker is now updated with all the procedures read from the logs<a name="line.104"></a>
+<span class="sourceLineNo">105</span>      if (tracker.isPartial()) {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>        tracker.setPartialFlag(false);<a name="line.106"></a>
+<span class="sourceLineNo">107</span>      }<a name="line.107"></a>
+<span class="sourceLineNo">108</span>      tracker.resetModified();<a name="line.108"></a>
+<span class="sourceLineNo">109</span>    } finally {<a name="line.109"></a>
+<span class="sourceLineNo">110</span>      tracker.setKeepDeletes(false);<a name="line.110"></a>
+<span class="sourceLineNo">111</span>    }<a name="line.111"></a>
+<span class="sourceLineNo">112</span>  }<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span>  public static void writeHeader(OutputStream stream, ProcedureWALHeader header)<a name="line.114"></a>
+<span class="sourceLineNo">115</span>      throws IOException {<a name="line.115"></a>
+<span class="sourceLineNo">116</span>    header.writeDelimitedTo(stream);<a name="line.116"></a>
+<span class="sourceLineNo">117</span>  }<a name="line.117"></a>
+<span class="sourceLineNo">118</span><a name="line.118"></a>
+<span class="sourceLineNo">119</span>  /*<a name="line.119"></a>
+<span class="sourceLineNo">120</span>   * +-----------------+<a name="line.120"></a>
+<span class="sourceLineNo">121</span>   * | END OF WAL DATA | &lt;---+<a name="line.121"></a>
+<span class="sourceLineNo">122</span>   * +-----------------+     |<a name="line.122"></a>
+<span class="sourceLineNo">123</span>   * |                 |     |<a name="line.123"></a>
+<span class="sourceLineNo">124</span>   * |     Tracker     |     |<a name="line.124"></a>
+<span class="sourceLineNo">125</span>   * |                 |     |<a name="line.125"></a>
+<span class="sourceLineNo">126</span>   * +-----------------+     |<a name="line.126"></a>
+<span class="sourceLineNo">127</span>   * |     version     |     |<a name="line.127"></a>
+<span class="sourceLineNo">128</span>   * +-----------------+     |<a name="line.128"></a>
+<span class="sourceLineNo">129</span>   * |  TRAILER_MAGIC  |     |<a name="line.129"></a>
+<span class="sourceLineNo">130</span>   * +-----------------+     |<a name="line.130"></a>
+<span class="sourceLineNo">131</span>   * |      offset     |-----+<a name="line.131"></a>
+<span class="sourceLineNo">132</span>   * +-----------------+<a name="line.132"></a>
+<span class="sourceLineNo">133</span>   */<a name="line.133"></a>
+<span class="sourceLineNo">134</span>  public static long writeTrailer(FSDataOutputStream stream, ProcedureStoreTracker tracker)<a name="line.134"></a>
+<span class="sourceLineNo">135</span>      throws IOException {<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    long offset = stream.getPos();<a name="line.136"></a>
+<span class="sourceLineNo">137</span><a name="line.137"></a>
+<span class="sourceLineNo">138</span>    // Write EOF Entry<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    ProcedureWALEntry.newBuilder()<a name="line.139"></a>
+<span class="sourceLineNo">140</span>      .setType(ProcedureWALEntry.Type.PROCEDURE_WAL_EOF)<a name="line.140"></a>
+<span class="sourceLineNo">141</span>      .build().writeDelimitedTo(stream);<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span>    // Write Tracker<a name="line.143"></a>
+<span class="sourceLineNo">144</span>    tracker.toProto().writeDelimitedTo(stream);<a name="line.144"></a>
+<span class="sourceLineNo">145</span><a name="line.145"></a>
+<span class="sourceLineNo">146</span>    stream.write(TRAILER_VERSION);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    StreamUtils.writeLong(stream, TRAILER_MAGIC);<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    StreamUtils.writeLong(stream, offset);<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    return stream.getPos() - offset;<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  public static ProcedureWALHeader readHeader(InputStream stream)<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    ProcedureWALHeader header;<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    try {<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      header = ProcedureWALHeader.parseDelimitedFrom(stream);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    } catch (InvalidProtocolBufferException e) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      throw new InvalidWALDataException(e);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>    if (header == null) {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      throw new InvalidWALDataException("No data available to read the Header");<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span>    if (header.getVersion() &lt; 0 || header.getVersion() != HEADER_VERSION) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      throw new InvalidWALDataException("Invalid Header version. got " + header.getVersion() +<a name="line.166"></a>
+<span class="sourceLineNo">167</span>          " expected " + HEADER_VERSION);<a name="line.167"></a>
+<span class="sourceLineNo">168</span>    }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>    if (header.getType() &lt; 0 || header.getType() &gt; LOG_TYPE_MAX_VALID) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>      throw new InvalidWALDataException("Invalid header type. got " + header.getType());<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
+<span class="sourceLineNo">173</span><a name="line.173"></a>
+<span class="sourceLineNo">174</span>    return header;<a name="line.174"></a>
+<span class="sourceLineNo">175</span>  }<a name="line.175"></a>
+<span class="sourceLineNo">176</span><a name="line.176"></a>
+<span class="sourceLineNo">177</span>  public static ProcedureWALTrailer readTrailer(FSDataInputStream stream, long startPos, long size)<a name="line.177"></a>
+<span class="sourceLineNo">178</span>      throws IOException {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>    // Beginning of the Trailer Jump. 17 = 1 byte version + 8 byte magic + 8 byte offset<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    long trailerPos = size - 17;<a name="line.180"></a>
 <span class="sourceLineNo">181</span><a name="line.181"></a>
-<span class="sourceLineNo">182</span>    long magic = StreamUtils.readLong(stream);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (magic != TRAILER_MAGIC) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.184"></a>
-<span class="sourceLineNo">185</span>          " expected " + TRAILER_MAGIC);<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    stream.seek(trailerOffset);<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      .setVersion(version)<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      .setTrackerPos(stream.getPos())<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      .build();<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    return trailer;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
-<span class="sourceLineNo">202</span><a name="line.202"></a>
-<span class="sourceLineNo">203</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  }<a name="line.205"></a>
+<span class="sourceLineNo">182</span>    if (trailerPos &lt; startPos) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>      throw new InvalidWALDataException("Missing trailer: size=" + size + " startPos=" + startPos);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span>    stream.seek(trailerPos);<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    int version = stream.read();<a name="line.187"></a>
+<span class="sourceLineNo">188</span>    if (version != TRAILER_VERSION) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>      throw new InvalidWALDataException("Invalid Trailer version. got " + version +<a name="line.189"></a>
+<span class="sourceLineNo">190</span>          " expected " + TRAILER_VERSION);<a name="line.190"></a>
+<span class="sourceLineNo">191</span>    }<a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span>    long magic = StreamUtils.readLong(stream);<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    if (magic != TRAILER_MAGIC) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>      throw new InvalidWALDataException("Invalid Trailer magic. got " + magic +<a name="line.195"></a>
+<span class="sourceLineNo">196</span>          " expected " + TRAILER_MAGIC);<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    }<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span>    long trailerOffset = StreamUtils.readLong(stream);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>    stream.seek(trailerOffset);<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span>    ProcedureWALEntry entry = readEntry(stream);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    if (entry.getType() != ProcedureWALEntry.Type.PROCEDURE_WAL_EOF) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      throw new InvalidWALDataException("Invalid Trailer begin");<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    }<a name="line.205"></a>
 <span class="sourceLineNo">206</span><a name="line.206"></a>
-<span class="sourceLineNo">207</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    builder.setType(type);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.211"></a>
-<span class="sourceLineNo">212</span>    if (subprocs != null) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      }<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    }<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    builder.build().writeDelimitedTo(slot);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>  }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      throws IOException {<a name="line.221"></a>
-<span class="sourceLineNo">222</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.225"></a>
-<span class="sourceLineNo">226</span>      throws IOException {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.227"></a>
-<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
-<span class="sourceLineNo">229</span><a name="line.229"></a>
-<span class="sourceLineNo">230</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      throws IOException {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>  }<a name="line.233"></a>
-<span class="sourceLineNo">234</span><a name="line.234"></a>
-<span class="sourceLineNo">235</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      throws IOException {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.237"></a>
-<span class="sourceLineNo">238</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.238"></a>
-<span class="sourceLineNo">239</span>    builder.setProcId(procId);<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    builder.build().writeDelimitedTo(slot);<a name="line.240"></a>
-<span class="sourceLineNo">241</span>  }<a name="line.241"></a>
-<span class="sourceLineNo">242</span><a name="line.242"></a>
-<span class="sourceLineNo">243</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      throws IOException {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.245"></a>
-<span class="sourceLineNo">246</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    builder.setProcId(proc.getProcId());<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    if (subprocs != null) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        builder.addChildId(subprocs[i]);<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    builder.build().writeDelimitedTo(slot);<a name="line.254"></a>
-<span class="sourceLineNo">255</span>  }<a name="line.255"></a>
-<span class="sourceLineNo">256</span>}<a name="line.256"></a>
+<span class="sourceLineNo">207</span>    ProcedureWALTrailer trailer = ProcedureWALTrailer.newBuilder()<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      .setVersion(version)<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      .setTrackerPos(stream.getPos())<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      .build();<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    return trailer;<a name="line.211"></a>
+<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span>  public static ProcedureWALEntry readEntry(InputStream stream) throws IOException {<a name="line.214"></a>
+<span class="sourceLineNo">215</span>    return ProcedureWALEntry.parseDelimitedFrom(stream);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>  }<a name="line.216"></a>
+<span class="sourceLineNo">217</span><a name="line.217"></a>
+<span class="sourceLineNo">218</span>  public static void writeEntry(ByteSlot slot, ProcedureWALEntry.Type type,<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs) throws IOException {<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    builder.setType(type);<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    if (subprocs != null) {<a name="line.223"></a>
+<span class="sourceLineNo">224</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.224"></a>
+<span class="sourceLineNo">225</span>        builder.addProcedure(ProcedureUtil.convertToProtoProcedure(subprocs[i]));<a name="line.225"></a>
+<span class="sourceLineNo">226</span>      }<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    }<a name="line.227"></a>
+<span class="sourceLineNo">228</span>    builder.build().writeDelimitedTo(slot);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
+<span class="sourceLineNo">230</span><a name="line.230"></a>
+<span class="sourceLineNo">231</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.231"></a>
+<span class="sourceLineNo">232</span>      throws IOException {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INIT, proc, null);<a name="line.233"></a>
+<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span>  public static void writeInsert(ByteSlot slot, Procedure&lt;?&gt; proc, Procedure&lt;?&gt;[] subprocs)<a name="line.236"></a>
+<span class="sourceLineNo">237</span>      throws IOException {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_INSERT, proc, subprocs);<a name="line.238"></a>
+<span class="sourceLineNo">239</span>  }<a name="line.239"></a>
+<span class="sourceLineNo">240</span><a name="line.240"></a>
+<span class="sourceLineNo">241</span>  public static void writeUpdate(ByteSlot slot, Procedure&lt;?&gt; proc)<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      throws IOException {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>    writeEntry(slot, ProcedureWALEntry.Type.PROCEDURE_WAL_UPDATE, proc, null);<a name="line.243"></a>
+<span class="sourceLineNo">244</span>  }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span>  public static void writeDelete(ByteSlot slot, long procId)<a name="line.246"></a>
+<span class="sourceLineNo">247</span>      throws IOException {<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.248"></a>
+<span class="sourceLineNo">249</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    builder.setProcId(procId);<a name="line.250"></a>
+<span class="sourceLineNo">251</span>    builder.build().writeDelimitedTo(slot);<a name="line.251"></a>
+<span class="sourceLineNo">252</span>  }<a name="line.252"></a>
+<span class="sourceLineNo">253</span><a name="line.253"></a>
+<span class="sourceLineNo">254</span>  public static void writeDelete(ByteSlot slot, Procedure&lt;?&gt; proc, long[] subprocs)<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      throws IOException {<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    final ProcedureWALEntry.Builder builder = ProcedureWALEntry.newBuilder();<a name="line.256"></a>
+<span class="sourceLineNo">257</span>    builder.setType(ProcedureWALEntry.Type.PROCEDURE_WAL_DELETE);<a name="line.257"></a>
+<span class="sourceLineNo">258</span>    builder.setProcId(proc.getProcId());<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    if (subprocs != null) {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      builder.addProcedure(ProcedureUtil.convertToProtoProcedure(proc));<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      for (int i = 0; i &lt; subprocs.length; ++i) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span>        builder.addChildId(subprocs[i]);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>      }<a name="line.263"></a>
+<span class="sourceLineNo">264</span>    }<a name="line.264"></a>
+<span class="sourceLineNo">265</span>    builder.build().writeDelimitedTo(slot);<a name="line.265"></a>
+<span class="sourceLineNo">266</span>  }<a name="line.266"></a>
+<span class="sourceLineNo">267</span>}<a name="line.267"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
index 797fe33..2d7fd0b 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
@@ -142,100 +142,99 @@
 <span class="sourceLineNo">134</span>      }<a name="line.134"></a>
 <span class="sourceLineNo">135</span>      procedureMap.merge(localProcedureMap);<a name="line.135"></a>
 <span class="sourceLineNo">136</span>    }<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    if (localTracker.isPartial()) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span>      localTracker.setPartialFlag(false);<a name="line.138"></a>
-<span class="sourceLineNo">139</span>    }<a name="line.139"></a>
-<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span>  public void finish() throws IOException {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    // notify the loader about the max proc ID<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    loader.setMaxProcId(maxProcId);<a name="line.144"></a>
-<span class="sourceLineNo">145</span><a name="line.145"></a>
-<span class="sourceLineNo">146</span>    // build the procedure execution tree. When building we will verify that whether a procedure is<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    // valid.<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    WALProcedureTree tree = WALProcedureTree.build(procedureMap.getProcedures());<a name="line.148"></a>
-<span class="sourceLineNo">149</span>    loader.load(tree.getValidProcs());<a name="line.149"></a>
-<span class="sourceLineNo">150</span>    loader.handleCorrupted(tree.getCorruptedProcs());<a name="line.150"></a>
-<span class="sourceLineNo">151</span>  }<a name="line.151"></a>
-<span class="sourceLineNo">152</span><a name="line.152"></a>
-<span class="sourceLineNo">153</span>  private void setDeletedIfPartial(ProcedureStoreTracker tracker, long procId) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    if (tracker.isPartial()) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span>      tracker.setDeleted(procId, true);<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    }<a name="line.156"></a>
-<span class="sourceLineNo">157</span>  }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>  private void insertIfPartial(ProcedureStoreTracker tracker, ProcedureProtos.Procedure proc) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    if (tracker.isPartial()) {<a name="line.160"></a>
-<span class="sourceLineNo">161</span>      tracker.insert(proc.getProcId());<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
-<span class="sourceLineNo">163</span>  }<a name="line.163"></a>
-<span class="sourceLineNo">164</span><a name="line.164"></a>
-<span class="sourceLineNo">165</span>  private void loadProcedure(ProcedureWALEntry entry, ProcedureProtos.Procedure proc) {<a name="line.165"></a>
-<span class="sourceLineNo">166</span>    maxProcId = Math.max(maxProcId, proc.getProcId());<a name="line.166"></a>
-<span class="sourceLineNo">167</span>    if (isRequired(proc.getProcId())) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>      LOG.trace("Read {} entry {}", entry.getType(), proc.getProcId());<a name="line.168"></a>
-<span class="sourceLineNo">169</span>      localProcedureMap.add(proc);<a name="line.169"></a>
-<span class="sourceLineNo">170</span>      insertIfPartial(tracker, proc);<a name="line.170"></a>
-<span class="sourceLineNo">171</span>    }<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    insertIfPartial(localTracker, proc);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>  private void readInitEntry(ProcedureWALEntry entry) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  }<a name="line.178"></a>
-<span class="sourceLineNo">179</span><a name="line.179"></a>
-<span class="sourceLineNo">180</span>  private void readInsertEntry(ProcedureWALEntry entry) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    assert entry.getProcedureCount() &gt;= 1 : "Expected one or more procedures";<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    for (int i = 1; i &lt; entry.getProcedureCount(); ++i) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      loadProcedure(entry, entry.getProcedure(i));<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    }<a name="line.185"></a>
-<span class="sourceLineNo">186</span>  }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span>  private void readUpdateEntry(ProcedureWALEntry entry) {<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.190"></a>
-<span class="sourceLineNo">191</span>  }<a name="line.191"></a>
-<span class="sourceLineNo">192</span><a name="line.192"></a>
-<span class="sourceLineNo">193</span>  private void readDeleteEntry(ProcedureWALEntry entry) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    assert entry.hasProcId() : "expected ProcID";<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span>    if (entry.getChildIdCount() &gt; 0) {<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.197"></a>
-<span class="sourceLineNo">198</span><a name="line.198"></a>
-<span class="sourceLineNo">199</span>      // update the parent procedure<a name="line.199"></a>
-<span class="sourceLineNo">200</span>      loadProcedure(entry, entry.getProcedure(0));<a name="line.200"></a>
-<span class="sourceLineNo">201</span><a name="line.201"></a>
-<span class="sourceLineNo">202</span>      // remove the child procedures of entry.getProcId()<a name="line.202"></a>
-<span class="sourceLineNo">203</span>      for (int i = 0, count = entry.getChildIdCount(); i &lt; count; ++i) {<a name="line.203"></a>
-<span class="sourceLineNo">204</span>        deleteEntry(entry.getChildId(i));<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      }<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    } else {<a name="line.206"></a>
-<span class="sourceLineNo">207</span>      assert entry.getProcedureCount() == 0 : "Expected no procedures";<a name="line.207"></a>
-<span class="sourceLineNo">208</span><a name="line.208"></a>
-<span class="sourceLineNo">209</span>      // delete the procedure<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      deleteEntry(entry.getProcId());<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
-<span class="sourceLineNo">213</span><a name="line.213"></a>
-<span class="sourceLineNo">214</span>  private void deleteEntry(final long procId) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>    LOG.trace("delete entry {}", procId);<a name="line.215"></a>
-<span class="sourceLineNo">216</span>    maxProcId = Math.max(maxProcId, procId);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>    localProcedureMap.remove(procId);<a name="line.217"></a>
-<span class="sourceLineNo">218</span>    assert !procedureMap.contains(procId);<a name="line.218"></a>
-<span class="sourceLineNo">219</span>    setDeletedIfPartial(tracker, procId);<a name="line.219"></a>
-<span class="sourceLineNo">220</span>    setDeletedIfPartial(localTracker, procId);<a name="line.220"></a>
-<span class="sourceLineNo">221</span>  }<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>  private boolean isDeleted(long procId) {<a name="line.223"></a>
-<span class="sourceLineNo">224</span>    return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;<a name="line.224"></a>
-<span class="sourceLineNo">225</span>  }<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>  private boolean isRequired(long procId) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span>    return !isDeleted(procId) &amp;&amp; !procedureMap.contains(procId);<a name="line.228"></a>
-<span class="sourceLineNo">229</span>  }<a name="line.229"></a>
-<span class="sourceLineNo">230</span>}<a name="line.230"></a>
+<span class="sourceLineNo">137</span>    // Do not reset the partial flag for local tracker, as here the local tracker only know the<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    // procedures which are modified in this file.<a name="line.138"></a>
+<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span>  public void finish() throws IOException {<a name="line.141"></a>
+<span class="sourceLineNo">142</span>    // notify the loader about the max proc ID<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    loader.setMaxProcId(maxProcId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span>    // build the procedure execution tree. When building we will verify that whether a procedure is<a name="line.145"></a>
+<span class="sourceLineNo">146</span>    // valid.<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    WALProcedureTree tree = WALProcedureTree.build(procedureMap.getProcedures());<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    loader.load(tree.getValidProcs());<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    loader.handleCorrupted(tree.getCorruptedProcs());<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  private void setDeletedIfPartial(ProcedureStoreTracker tracker, long procId) {<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    if (tracker.isPartial()) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>      tracker.setDeleted(procId, true);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    }<a name="line.155"></a>
+<span class="sourceLineNo">156</span>  }<a name="line.156"></a>
+<span class="sourceLineNo">157</span><a name="line.157"></a>
+<span class="sourceLineNo">158</span>  private void insertIfPartial(ProcedureStoreTracker tracker, ProcedureProtos.Procedure proc) {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    if (tracker.isPartial()) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>      tracker.insert(proc.getProcId());<a name="line.160"></a>
+<span class="sourceLineNo">161</span>    }<a name="line.161"></a>
+<span class="sourceLineNo">162</span>  }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span>  private void loadProcedure(ProcedureWALEntry entry, ProcedureProtos.Procedure proc) {<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    maxProcId = Math.max(maxProcId, proc.getProcId());<a name="line.165"></a>
+<span class="sourceLineNo">166</span>    if (isRequired(proc.getProcId())) {<a name="line.166"></a>
+<span class="sourceLineNo">167</span>      LOG.trace("Read {} entry {}", entry.getType(), proc.getProcId());<a name="line.167"></a>
+<span class="sourceLineNo">168</span>      localProcedureMap.add(proc);<a name="line.168"></a>
+<span class="sourceLineNo">169</span>      insertIfPartial(tracker, proc);<a name="line.169"></a>
+<span class="sourceLineNo">170</span>    }<a name="line.170"></a>
+<span class="sourceLineNo">171</span>    insertIfPartial(localTracker, proc);<a name="line.171"></a>
+<span class="sourceLineNo">172</span>  }<a name="line.172"></a>
+<span class="sourceLineNo">173</span><a name="line.173"></a>
+<span class="sourceLineNo">174</span>  private void readInitEntry(ProcedureWALEntry entry) {<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.175"></a>
+<span class="sourceLineNo">176</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.176"></a>
+<span class="sourceLineNo">177</span>  }<a name="line.177"></a>
+<span class="sourceLineNo">178</span><a name="line.178"></a>
+<span class="sourceLineNo">179</span>  private void readInsertEntry(ProcedureWALEntry entry) {<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    assert entry.getProcedureCount() &gt;= 1 : "Expected one or more procedures";<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.181"></a>
+<span class="sourceLineNo">182</span>    for (int i = 1; i &lt; entry.getProcedureCount(); ++i) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>      loadProcedure(entry, entry.getProcedure(i));<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
+<span class="sourceLineNo">185</span>  }<a name="line.185"></a>
+<span class="sourceLineNo">186</span><a name="line.186"></a>
+<span class="sourceLineNo">187</span>  private void readUpdateEntry(ProcedureWALEntry entry) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span>    assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.188"></a>
+<span class="sourceLineNo">189</span>    loadProcedure(entry, entry.getProcedure(0));<a name="line.189"></a>
+<span class="sourceLineNo">190</span>  }<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>  private void readDeleteEntry(ProcedureWALEntry entry) {<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    assert entry.hasProcId() : "expected ProcID";<a name="line.193"></a>
+<span class="sourceLineNo">194</span><a name="line.194"></a>
+<span class="sourceLineNo">195</span>    if (entry.getChildIdCount() &gt; 0) {<a name="line.195"></a>
+<span class="sourceLineNo">196</span>      assert entry.getProcedureCount() == 1 : "Expected only one procedure";<a name="line.196"></a>
+<span class="sourceLineNo">197</span><a name="line.197"></a>
+<span class="sourceLineNo">198</span>      // update the parent procedure<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      loadProcedure(entry, entry.getProcedure(0));<a name="line.199"></a>
+<span class="sourceLineNo">200</span><a name="line.200"></a>
+<span class="sourceLineNo">201</span>      // remove the child procedures of entry.getProcId()<a name="line.201"></a>
+<span class="sourceLineNo">202</span>      for (int i = 0, count = entry.getChildIdCount(); i &lt; count; ++i) {<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        deleteEntry(entry.getChildId(i));<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      }<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    } else {<a name="line.205"></a>
+<span class="sourceLineNo">206</span>      assert entry.getProcedureCount() == 0 : "Expected no procedures";<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>      // delete the procedure<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      deleteEntry(entry.getProcId());<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    }<a name="line.210"></a>
+<span class="sourceLineNo">211</span>  }<a name="line.211"></a>
+<span class="sourceLineNo">212</span><a name="line.212"></a>
+<span class="sourceLineNo">213</span>  private void deleteEntry(final long procId) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>    LOG.trace("delete entry {}", procId);<a name="line.214"></a>
+<span class="sourceLineNo">215</span>    maxProcId = Math.max(maxProcId, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>    localProcedureMap.remove(procId);<a name="line.216"></a>
+<span class="sourceLineNo">217</span>    assert !procedureMap.contains(procId);<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    setDeletedIfPartial(tracker, procId);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>    setDeletedIfPartial(localTracker, procId);<a name="line.219"></a>
+<span class="sourceLineNo">220</span>  }<a name="line.220"></a>
+<span class="sourceLineNo">221</span><a name="line.221"></a>
+<span class="sourceLineNo">222</span>  private boolean isDeleted(long procId) {<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    return tracker.isDeleted(procId) == ProcedureStoreTracker.DeleteState.YES;<a name="line.223"></a>
+<span class="sourceLineNo">224</span>  }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>  private boolean isRequired(long procId) {<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    return !isDeleted(procId) &amp;&amp; !procedureMap.contains(procId);<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
+<span class="sourceLineNo">229</span>}<a name="line.229"></a>
 
 
 


[24/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class=

<TRUNCATED>

[50/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/index-all.html
----------------------------------------------------------------------
diff --git a/devapidocs/index-all.html b/devapidocs/index-all.html
index a9d9513..3f2ac5e 100644
--- a/devapidocs/index-all.html
+++ b/devapidocs/index-all.html
@@ -22753,6 +22753,8 @@
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/namespace/NamespaceTableAndRegionInfo.html#decrementRegionCountForTable-org.apache.hadoop.hbase.TableName-int-">decrementRegionCountForTable(TableName, int)</a></span> - Method in class org.apache.hadoop.hbase.namespace.<a href="org/apache/hadoop/hbase/namespace/NamespaceTableAndRegionInfo.html" title="class in org.apache.hadoop.hbase.namespace">NamespaceTableAndRegionInfo</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#decrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">decrementUgiReference(UserGroupInformation)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/HRegion.html#decrMemStoreSize-org.apache.hadoop.hbase.regionserver.MemStoreSize-">decrMemStoreSize(MemStoreSize)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/HRegion.html#decrMemStoreSize-long-long-long-">decrMemStoreSize(long, long, long)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a></dt>
@@ -33723,6 +33725,8 @@
 </dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.html#fScan">fScan</a></span> - Variable in class org.apache.hadoop.hbase.rest.<a href="org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.html" title="class in org.apache.hadoop.hbase.rest">MetricsRESTSourceImpl</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#fsCreatedListener">fsCreatedListener</a></span> - Variable in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><a href="org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.html" title="class in org.apache.hadoop.hbase.io"><span class="typeNameLink">FSDataInputStreamWrapper</span></a> - Class in <a href="org/apache/hadoop/hbase/io/package-summary.html">org.apache.hadoop.hbase.io</a></dt>
 <dd>
 <div class="block">Wrapper for input stream(s) that takes care of the interaction of FS and HBase checksums,
@@ -60511,6 +60515,8 @@
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/client/MetricsConnection.html#incrementTracker">incrementTracker</a></span> - Variable in class org.apache.hadoop.hbase.client.<a href="org/apache/hadoop/hbase/client/MetricsConnection.html" title="class in org.apache.hadoop.hbase.client">MetricsConnection</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#incrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">incrementUgiReference(UserGroupInformation)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.html#incrementWrittenBytes-long-">incrementWrittenBytes(long)</a></span> - Method in interface org.apache.hadoop.hbase.regionserver.wal.<a href="org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.html" title="interface in org.apache.hadoop.hbase.regionserver.wal">MetricsWALSource</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.html#incrementWrittenBytes-long-">incrementWrittenBytes(long)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.wal.<a href="org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.html" title="class in org.apache.hadoop.hbase.regionserver.wal">MetricsWALSourceImpl</a></dt>
@@ -65652,6 +65658,8 @@
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/ScanInfo.html#isUsePread--">isUsePread()</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/ScanInfo.html" title="class in org.apache.hadoop.hbase.regionserver">ScanInfo</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#isUserReferenced-org.apache.hadoop.security.UserGroupInformation-">isUserReferenced(UserGroupInformation)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/quotas/QuotaTableUtil.html#isUserRowKey-byte:A-">isUserRowKey(byte[])</a></span> - Static method in class org.apache.hadoop.hbase.quotas.<a href="org/apache/hadoop/hbase/quotas/QuotaTableUtil.html" title="class in org.apache.hadoop.hbase.quotas">QuotaTableUtil</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/querymatcher/CompactionScanQueryMatcher.html#isUserScan--">isUserScan()</a></span> - Method in class org.apache.hadoop.hbase.regionserver.querymatcher.<a href="org/apache/hadoop/hbase/regionserver/querymatcher/CompactionScanQueryMatcher.html" title="class in org.apache.hadoop.hbase.regionserver.querymatcher">CompactionScanQueryMatcher</a></dt>
@@ -68208,7 +68216,10 @@
 <div class="block">Called by the ProcedureStore.load() every time a set of procedures are ready to be executed.</div>
 </dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#load-java.util.Iterator-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.Loader-">load(Iterator&lt;ProcedureWALFile&gt;, ProcedureStoreTracker, ProcedureWALFormat.Loader)</a></span> - Static method in class org.apache.hadoop.hbase.procedure2.store.wal.<a href="org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat</a></dt>
-<dd>&nbsp;</dd>
+<dd>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.</div>
+</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#load-org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoader-">load(ProcedureStore.ProcedureLoader)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.wal.<a href="org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/wal/RingBufferTruck.html#load-org.apache.hadoop.hbase.regionserver.wal.FSWALEntry-">load(FSWALEntry)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.wal.<a href="org/apache/hadoop/hbase/regionserver/wal/RingBufferTruck.html" title="class in org.apache.hadoop.hbase.regionserver.wal">RingBufferTruck</a></dt>
@@ -69631,6 +69642,8 @@
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.html#LOG">LOG</a></span> - Static variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/RemoteProcedureDispatcher.html" title="class in org.apache.hadoop.hbase.procedure2">RemoteProcedureDispatcher</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/RootProcedureState.html#LOG">LOG</a></span> - Static variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/StateMachineProcedure.html#LOG">LOG</a></span> - Static variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/StateMachineProcedure.html" title="class in org.apache.hadoop.hbase.procedure2">StateMachineProcedure</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/StoppableThread.html#LOG">LOG</a></span> - Static variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/StoppableThread.html" title="class in org.apache.hadoop.hbase.procedure2">StoppableThread</a></dt>
@@ -102459,11 +102472,16 @@ service.</div>
 <dd>
 <div class="block">This method is used when restarting where we need to rebuild the ProcedureStoreTracker.</div>
 </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfDeletedByThem-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfDeletedByThem(ProcedureStoreTracker)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.<a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a></dt>
+<dd>
+<div class="block">For the global tracker, we will use this method to build the holdingCleanupTracker, as the
+ modified flags will be cleared after rolling so we only need to test the deleted flags.</div>
+</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-">setDeletedIfModified(long...)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.<a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a></dt>
 <dd>
 <div class="block">Set the given bit for the procId to delete if it was modified before.</div>
 </dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-">setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.<a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a></dt>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfModifiedInBoth(ProcedureStoreTracker)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.<a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a></dt>
 <dd>
 <div class="block">Similar with <a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-"><code>ProcedureStoreTracker.setDeletedIfModified(long...)</code></a>, but here the <code>procId</code> are given by
  the <code>tracker</code>.</div>
@@ -102474,6 +102492,8 @@ service.</div>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/StoreFileReader.html#setDeleteFamilyBloomFilterFaulty--">setDeleteFamilyBloomFilterFaulty()</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/StoreFileReader.html" title="class in org.apache.hadoop.hbase.regionserver">StoreFileReader</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeleteIf-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-java.util.function.BiFunction-">setDeleteIf(ProcedureStoreTracker, BiFunction&lt;BitSetNode, Long, Boolean&gt;)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.store.<a href="org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.html#setDeprecatedProperties--">setDeprecatedProperties()</a></span> - Method in class org.apache.hadoop.hbase.coprocessor.<a href="org/apache/hadoop/hbase/coprocessor/ReadOnlyConfiguration.html" title="class in org.apache.hadoop.hbase.coprocessor">ReadOnlyConfiguration</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/monitoring/MonitoredTask.html#setDescription-java.lang.String-">setDescription(String)</a></span> - Method in interface org.apache.hadoop.hbase.monitoring.<a href="org/apache/hadoop/hbase/monitoring/MonitoredTask.html" title="interface in org.apache.hadoop.hbase.monitoring">MonitoredTask</a></dt>
@@ -102966,6 +102986,8 @@ service.</div>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/quotas/TimeBasedLimiter.html#setFromTimedQuota-org.apache.hadoop.hbase.quotas.RateLimiter-org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota-">setFromTimedQuota(RateLimiter, QuotaProtos.TimedQuota)</a></span> - Static method in class org.apache.hadoop.hbase.quotas.<a href="org/apache/hadoop/hbase/quotas/TimeBasedLimiter.html" title="class in org.apache.hadoop.hbase.quotas">TimeBasedLimiter</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#setFsCreatedListener-java.util.function.Consumer-">setFsCreatedListener(Consumer&lt;HRegion&gt;)</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/util/CommonFSUtils.html#setFsDefault-org.apache.hadoop.conf.Configuration-org.apache.hadoop.fs.Path-">setFsDefault(Configuration, Path)</a></span> - Static method in class org.apache.hadoop.hbase.util.<a href="org/apache/hadoop/hbase/util/CommonFSUtils.html" title="class in org.apache.hadoop.hbase.util">CommonFSUtils</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/metrics/BaseSource.html#setGauge-java.lang.String-long-">setGauge(String, long)</a></span> - Method in interface org.apache.hadoop.hbase.metrics.<a href="org/apache/hadoop/hbase/metrics/BaseSource.html" title="interface in org.apache.hadoop.hbase.metrics">BaseSource</a></dt>
@@ -117897,6 +117919,8 @@ service.</div>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/security/User.html#ugi">ugi</a></span> - Variable in class org.apache.hadoop.hbase.security.<a href="org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a></dt>
 <dd>&nbsp;</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#ugiReferenceCounter">ugiReferenceCounter</a></span> - Variable in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html" title="class in org.apache.hadoop.hbase.regionserver">SecureBulkLoadManager</a></dt>
+<dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.html#unackedAppends">unackedAppends</a></span> - Variable in class org.apache.hadoop.hbase.regionserver.wal.<a href="org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.html" title="class in org.apache.hadoop.hbase.regionserver.wal">AsyncFSWAL</a></dt>
 <dd>&nbsp;</dd>
 <dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/util/UnsafeAvailChecker.html#unaligned">unaligned</a></span> - Static variable in class org.apache.hadoop.hbase.util.<a href="org/apache/hadoop/hbase/util/UnsafeAvailChecker.html" title="class in org.apache.hadoop.hbase.util">UnsafeAvailChecker</a></dt>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
index c4b3203..f11e74d 100644
--- a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
@@ -167,8 +167,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupState.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupState</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupRestoreConstants.BackupCommand.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupRestoreConstants.BackupCommand</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupState.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupState</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupPhase.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupPhase</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupType.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupType</span></a></li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
index 3110163..2ae1a4d 100644
--- a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
@@ -555,24 +555,24 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactionState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactionState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.Retry.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncRequestFutureImpl.Retry</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/ScannerCallable.MoreResults.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">ScannerCallable.MoreResults</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MasterSwitchType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MasterSwitchType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AbstractResponse.ResponseType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AbstractResponse.ResponseType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/IsolationLevel.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">IsolationLevel</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Consistency.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Consistency</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncProcessTask.SubmittedRows.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncProcessTask.SubmittedRows</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Durability</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MobCompactPartitionPolicy</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.Retry.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncRequestFutureImpl.Retry</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactionState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactionState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncProcessTask.SubmittedRows.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncProcessTask.SubmittedRows</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Scan.ReadType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Scan.ReadType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Durability</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RegionLocateType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RegionLocateType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/ScannerCallable.MoreResults.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">ScannerCallable.MoreResults</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/IsolationLevel.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">IsolationLevel</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
index e67b778..329f62f 100644
--- a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
@@ -201,8 +201,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.MetaTableOps.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">MetaTableMetrics.MetaTableOps</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/RegionObserver.MutationType.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">RegionObserver.MutationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.MetaTableOps.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">MetaTableMetrics.MetaTableOps</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/executor/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/executor/package-tree.html b/devapidocs/org/apache/hadoop/hbase/executor/package-tree.html
index ae0124e..ce950b0 100644
--- a/devapidocs/org/apache/hadoop/hbase/executor/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/executor/package-tree.html
@@ -104,8 +104,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.executor.<a href="../../../../../org/apache/hadoop/hbase/executor/ExecutorType.html" title="enum in org.apache.hadoop.hbase.executor"><span class="typeNameLink">ExecutorType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.executor.<a href="../../../../../org/apache/hadoop/hbase/executor/EventType.html" title="enum in org.apache.hadoop.hbase.executor"><span class="typeNameLink">EventType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.executor.<a href="../../../../../org/apache/hadoop/hbase/executor/ExecutorType.html" title="enum in org.apache.hadoop.hbase.executor"><span class="typeNameLink">ExecutorType</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
index 92f4246..3ed3629 100644
--- a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
@@ -183,14 +183,14 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/RegexStringComparator.EngineType.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">RegexStringComparator.EngineType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/CompareFilter.CompareOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">CompareFilter.CompareOp</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/BitComparator.BitwiseOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">BitComparator.BitwiseOp</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.SatisfiesCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.SatisfiesCode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/Filter.ReturnCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">Filter.ReturnCode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.Order.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.Order</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/CompareFilter.CompareOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">CompareFilter.CompareOp</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/BitComparator.BitwiseOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">BitComparator.BitwiseOp</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterWrapper.FilterRowRetCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterWrapper.FilterRowRetCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/RegexStringComparator.EngineType.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">RegexStringComparator.EngineType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
index f84acf2..6bd4567 100644
--- a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
@@ -274,12 +274,12 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/CacheConfig.ExternalBlockCaches.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">CacheConfig.ExternalBlockCaches</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/HFileBlock.Writer.State.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">HFileBlock.Writer.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/Cacheable.MemoryType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">Cacheable.MemoryType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.BlockCategory.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType.BlockCategory</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/CacheConfig.ExternalBlockCaches.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">CacheConfig.ExternalBlockCaches</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockPriority.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockPriority</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/Cacheable.MemoryType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">Cacheable.MemoryType</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/ipc/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/ipc/package-tree.html b/devapidocs/org/apache/hadoop/hbase/ipc/package-tree.html
index 7557e4f..df0adf0 100644
--- a/devapidocs/org/apache/hadoop/hbase/ipc/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/ipc/package-tree.html
@@ -353,9 +353,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
+<li type="circle">org.apache.hadoop.hbase.ipc.<a href="../../../../../org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.SourceStorage.html" title="enum in org.apache.hadoop.hbase.ipc"><span class="typeNameLink">MetricsHBaseServerSourceFactoryImpl.SourceStorage</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.ipc.<a href="../../../../../org/apache/hadoop/hbase/ipc/BufferCallBeforeInitHandler.BufferCallAction.html" title="enum in org.apache.hadoop.hbase.ipc"><span class="typeNameLink">BufferCallBeforeInitHandler.BufferCallAction</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.ipc.<a href="../../../../../org/apache/hadoop/hbase/ipc/CallEvent.Type.html" title="enum in org.apache.hadoop.hbase.ipc"><span class="typeNameLink">CallEvent.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.ipc.<a href="../../../../../org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceFactoryImpl.SourceStorage.html" title="enum in org.apache.hadoop.hbase.ipc"><span class="typeNameLink">MetricsHBaseServerSourceFactoryImpl.SourceStorage</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
index f1985a3..730b7cf 100644
--- a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
@@ -293,9 +293,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/TableSplit.Version.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">TableSplit.Version</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/SyncTable.SyncMapper.Counter.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">SyncTable.SyncMapper.Counter</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/TableSplit.Version.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">TableSplit.Version</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/RowCounter.RowCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">RowCounter.RowCounterMapper.Counters</span></a></li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html b/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
index 3fd5a93..0b7dbb1 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.305">CatalogJanitor.SplitParentFirstComparator</a>
+<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.307">CatalogJanitor.SplitParentFirstComparator</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
 implements <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&gt;</pre>
 <div class="block">Compare HRegionInfos in a way that has split parents sort BEFORE their
@@ -216,7 +216,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparat
 <ul class="blockListLast">
 <li class="blockList">
 <h4>rowEndKeyComparator</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;byte[]&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.306">rowEndKeyComparator</a></pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;byte[]&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.308">rowEndKeyComparator</a></pre>
 </li>
 </ul>
 </li>
@@ -233,7 +233,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparat
 <ul class="blockListLast">
 <li class="blockList">
 <h4>SplitParentFirstComparator</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.305">SplitParentFirstComparator</a>()</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.307">SplitParentFirstComparator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -250,7 +250,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparat
 <ul class="blockListLast">
 <li class="blockList">
 <h4>compare</h4>
-<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.308">compare</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;left,
+<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html#line.310">compare</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;left,
                    <a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;right)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.html b/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.html
index d054a89..2c5ccb5 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/CatalogJanitor.html
@@ -454,7 +454,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>getMergedRegionsAndSplitParents</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/util/Triple.html" title="class in org.apache.hadoop.hbase.util">Triple</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/
 hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.136">getMergedRegionsAndSplitParents</a>()
+<pre><a href="../../../../../org/apache/hadoop/hbase/util/Triple.html" title="class in org.apache.hadoop.hbase.util">Triple</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/
 hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.138">getMergedRegionsAndSplitParents</a>()
                                                                                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Scans hbase:meta and returns a number of scanned rows, and a map of merged
  regions, and an ordered map of split parents.</div>
@@ -473,7 +473,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>getMergedRegionsAndSplitParents</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/util/Triple.html" title="class in org.apache.hadoop.hbase.util">Triple</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/
 hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.151">getMergedRegionsAndSplitParents</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre><a href="../../../../../org/apache/hadoop/hbase/util/Triple.html" title="class in org.apache.hadoop.hbase.util">Triple</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/
 hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.153">getMergedRegionsAndSplitParents</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                                                                                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Scans hbase:meta and returns a number of scanned rows, and a map of merged
  regions, and an ordered map of split parents. if the given table name is
@@ -496,7 +496,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>cleanMergeRegion</h4>
-<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.198">cleanMergeRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;mergedRegion,
+<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.200">cleanMergeRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;mergedRegion,
                          <a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionA,
                          <a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionB)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -519,7 +519,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>scan</h4>
-<pre>int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.234">scan</a>()
+<pre>int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.236">scan</a>()
   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Run janitorial scan of catalog <code>hbase:meta</code> table looking for
  garbage to collect.</div>
@@ -537,7 +537,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>cleanParent</h4>
-<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.335">cleanParent</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;parent,
+<pre>boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.337">cleanParent</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;parent,
                     <a href="../../../../../org/apache/hadoop/hbase/client/Result.html" title="class in org.apache.hadoop.hbase.client">Result</a>&nbsp;rowContent)
              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">If daughters no longer hold reference to the parents, delete the parent.</div>
@@ -560,7 +560,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>hasNoReferences</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.372">hasNoReferences</a>(<a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;p)</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.374">hasNoReferences</a>(<a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;p)</pre>
 <dl>
 <dt><span class="paramLabel">Parameters:</span></dt>
 <dd><code>p</code> - A pair where the first boolean says whether or not the daughter
@@ -577,7 +577,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>checkDaughterInFs</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.386">checkDaughterInFs</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;parent,
+<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.388">checkDaughterInFs</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;parent,
                                         <a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;daughter)
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Checks if a daughter region -- either splitA or splitB -- still holds
@@ -601,7 +601,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockList">
 <li class="blockList">
 <h4>getTableDescriptor</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.429">getTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.431">getTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                                     throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/FileNotFoundException.html?is-external=true" title="class or interface in java.io">FileNotFoundException</a>,
                                            <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -617,7 +617,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/ScheduledChore.html" tit
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cleanMergeQualifier</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.441">cleanMergeQualifier</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;region)
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html#line.443">cleanMergeQualifier</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;region)
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Checks if the specified region has merge qualifiers, if so, try to clean
  them</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
index d81b9be..9fafb83 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
@@ -136,7 +136,7 @@
 </dl>
 <hr>
 <br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3054">HMaster.MasterStoppedException</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3056">HMaster.MasterStoppedException</a>
 extends <a href="../../../../../org/apache/hadoop/hbase/DoNotRetryIOException.html" title="class in org.apache.hadoop.hbase">DoNotRetryIOException</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
@@ -205,7 +205,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/DoNotRetryIOException.ht
 <ul class="blockListLast">
 <li class="blockList">
 <h4>MasterStoppedException</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html#line.3055">MasterStoppedException</a>()</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html#line.3057">MasterStoppedException</a>()</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
index db9a9cb..c7f8e03 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
@@ -105,7 +105,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>protected static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2473">HMaster.TableDescriptorGetter</a></pre>
+<pre>protected static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2475">HMaster.TableDescriptorGetter</a></pre>
 <div class="block">Implement to return TableDescriptor after pre-checks</div>
 </li>
 </ul>
@@ -150,7 +150,7 @@ var activeTableTab = "activeTableTab";
 <ul class="blockListLast">
 <li class="blockList">
 <h4>get</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html#line.2474">get</a>()
+<pre><a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html#line.2476">get</a>()
              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>


[31/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLin

<TRUNCATED>

[15/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">

<TRUNCATED>

[34/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelt

<TRUNCATED>

[37/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
index b7a2a38..6591d6c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.html
@@ -1789,2174 +1789,2176 @@
 <span class="sourceLineNo">1781</span>      LOG.debug("Master has not been initialized, don't run region normalizer.");<a name="line.1781"></a>
 <span class="sourceLineNo">1782</span>      return false;<a name="line.1782"></a>
 <span class="sourceLineNo">1783</span>    }<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span><a name="line.1784"></a>
-<span class="sourceLineNo">1785</span>    if (isInMaintenanceMode()) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span>      return false;<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span>    }<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span><a name="line.1789"></a>
-<span class="sourceLineNo">1790</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1791"></a>
-<span class="sourceLineNo">1792</span>      return false;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span>    }<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span><a name="line.1794"></a>
-<span class="sourceLineNo">1795</span>    synchronized (this.normalizer) {<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span>      // Don't run the normalizer concurrently<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1797"></a>
-<span class="sourceLineNo">1798</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1798"></a>
-<span class="sourceLineNo">1799</span><a name="line.1799"></a>
-<span class="sourceLineNo">1800</span>      Collections.shuffle(allEnabledTables);<a name="line.1800"></a>
+<span class="sourceLineNo">1784</span>    if (this.getServerManager().isClusterShutdown()) {<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span>      LOG.info("Cluster is shutting down, don't run region normalizer.");<a name="line.1785"></a>
+<span class="sourceLineNo">1786</span>      return false;<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span>    }<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span>    if (isInMaintenanceMode()) {<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span>      return false;<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span>    }<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span>      return false;<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span>    }<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span><a name="line.1796"></a>
+<span class="sourceLineNo">1797</span>    synchronized (this.normalizer) {<a name="line.1797"></a>
+<span class="sourceLineNo">1798</span>      // Don't run the normalizer concurrently<a name="line.1798"></a>
+<span class="sourceLineNo">1799</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1800"></a>
 <span class="sourceLineNo">1801</span><a name="line.1801"></a>
-<span class="sourceLineNo">1802</span>      for (TableName table : allEnabledTables) {<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span>        if (isInMaintenanceMode()) {<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span>          return false;<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span>        }<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span><a name="line.1807"></a>
-<span class="sourceLineNo">1808</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span>          continue;<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span>        }<a name="line.1814"></a>
-<span class="sourceLineNo">1815</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span>        if (plans != null) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span>          for (NormalizationPlan plan : plans) {<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span>              splitPlanCount++;<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span>              mergePlanCount++;<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span>            }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span>          }<a name="line.1824"></a>
-<span class="sourceLineNo">1825</span>        }<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span>      }<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span>    }<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span>    // Return true indicating a success.<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span>    return true;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span>  }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span>  /**<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span>   */<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span>  @Override<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span>  public String getClientIdAuditPrefix() {<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span>  }<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span><a name="line.1841"></a>
-<span class="sourceLineNo">1842</span>  /**<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span>   * Switch for the background CatalogJanitor thread.<a name="line.1843"></a>
-<span class="sourceLineNo">1844</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span>   * if disabled.<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span>   */<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span>  }<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span><a name="line.1851"></a>
-<span class="sourceLineNo">1852</span>  @Override<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span>  public long mergeRegions(<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span>      final RegionInfo[] regionsToMerge,<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span>      final boolean forcible,<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span>      final long nonceGroup,<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span>      final long nonce) throws IOException {<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span>    checkInitialized();<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span><a name="line.1859"></a>
-<span class="sourceLineNo">1860</span>    assert(regionsToMerge.length == 2);<a name="line.1860"></a>
+<span class="sourceLineNo">1802</span>      Collections.shuffle(allEnabledTables);<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span><a name="line.1803"></a>
+<span class="sourceLineNo">1804</span>      for (TableName table : allEnabledTables) {<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span>        if (isInMaintenanceMode()) {<a name="line.1805"></a>
+<span class="sourceLineNo">1806</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span>          return false;<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span>        }<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span><a name="line.1809"></a>
+<span class="sourceLineNo">1810</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span>          continue;<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span>        }<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span>        if (plans != null) {<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span>          for (NormalizationPlan plan : plans) {<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1821"></a>
+<span class="sourceLineNo">1822</span>              splitPlanCount++;<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span>              mergePlanCount++;<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span>            }<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span>          }<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span>      }<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span>    }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span>    // Return true indicating a success.<a name="line.1831"></a>
+<span class="sourceLineNo">1832</span>    return true;<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span>  }<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span><a name="line.1834"></a>
+<span class="sourceLineNo">1835</span>  /**<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span>   */<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span>  @Override<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span>  public String getClientIdAuditPrefix() {<a name="line.1839"></a>
+<span class="sourceLineNo">1840</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span>  }<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span><a name="line.1843"></a>
+<span class="sourceLineNo">1844</span>  /**<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span>   * Switch for the background CatalogJanitor thread.<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span>   * if disabled.<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span>   */<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1850"></a>
+<span class="sourceLineNo">1851</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span>  }<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span><a name="line.1853"></a>
+<span class="sourceLineNo">1854</span>  @Override<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span>  public long mergeRegions(<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span>      final RegionInfo[] regionsToMerge,<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span>      final boolean forcible,<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span>      final long nonceGroup,<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span>      final long nonce) throws IOException {<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span>    checkInitialized();<a name="line.1860"></a>
 <span class="sourceLineNo">1861</span><a name="line.1861"></a>
-<span class="sourceLineNo">1862</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span>    }<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span><a name="line.1866"></a>
-<span class="sourceLineNo">1867</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span>      throw new IOException (<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1869"></a>
-<span class="sourceLineNo">1870</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span>    }<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span><a name="line.1872"></a>
-<span class="sourceLineNo">1873</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span>      throw new MergeRegionException(<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span>    }<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span><a name="line.1877"></a>
-<span class="sourceLineNo">1878</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1878"></a>
-<span class="sourceLineNo">1879</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span>      @Override<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span>      protected void run() throws IOException {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span><a name="line.1883"></a>
-<span class="sourceLineNo">1884</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span><a name="line.1886"></a>
-<span class="sourceLineNo">1887</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span>          regionsToMerge, forcible));<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span><a name="line.1889"></a>
-<span class="sourceLineNo">1890</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span>      }<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span><a name="line.1892"></a>
-<span class="sourceLineNo">1893</span>      @Override<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span>      protected String getDescription() {<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span>        return "MergeTableProcedure";<a name="line.1895"></a>
-<span class="sourceLineNo">1896</span>      }<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span>    });<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span>  }<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span>  @Override<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1901"></a>
-<span class="sourceLineNo">1902</span>      final long nonceGroup, final long nonce)<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span>  throws IOException {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span>    checkInitialized();<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1905"></a>
-<span class="sourceLineNo">1906</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>      @Override<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>      protected void run() throws IOException {<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span><a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>        // Execute the operation asynchronously<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span><a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>      @Override<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span>      protected String getDescription() {<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>        return "SplitTableProcedure";<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>      }<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>    });<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>  // Replace with an async implementation from which you can get<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span>  // a success/failure result.<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>  @VisibleForTesting<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span><a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    RegionInfo hri;<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>    if (regionState != null) {<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>      hri = regionState.getRegion();<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>    } else {<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span><a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>    ServerName dest;<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>        : new ArrayList&lt;&gt;(1);<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>      LOG.info(<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>              + " because the server is in exclude list");<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>      destServerName = null;<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>    }<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>        "choosing a server at random");<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>      exclude.add(regionState.getServerName());<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>      if (dest == null) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>        return;<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span>      }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    } else {<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span>      if (dest == null) {<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>        return;<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>      }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span>      // TODO: What is this? I don't get it.<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>        // regions on master.<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>          + " because it should not be on master");<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>        return;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span>      }<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    }<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span><a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>      return;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>    }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>    // Now we can do the move<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span><a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>    try {<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      checkInitialized();<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>      if (this.cpHost != null) {<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>      }<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span>      // closed<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span><a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>      try {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        // Is this going to work? Will we throw exception on error?<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        future.get();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>        throw new HBaseIOException(e);<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>      }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span>      if (this.cpHost != null) {<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>    } catch (IOException ioe) {<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      if (ioe instanceof HBaseIOException) {<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        throw (HBaseIOException)ioe;<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      }<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>      throw new HBaseIOException(ioe);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span>  }<a name="line.2014"></a>
-<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>  @Override<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>  public long createTable(<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>      final TableDescriptor tableDescriptor,<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>      final byte [][] splitKeys,<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>      final long nonceGroup,<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span>      final long nonce) throws IOException {<a name="line.2021"></a>
-<span class="sourceLineNo">2022</span>    checkInitialized();<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span><a name="line.2023"></a>
-<span class="sourceLineNo">2024</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span><a name="line.2026"></a>
-<span class="sourceLineNo">2027</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2027"></a>
-<span class="sourceLineNo">2028</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2028"></a>
-<span class="sourceLineNo">2029</span><a name="line.2029"></a>
-<span class="sourceLineNo">2030</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span>      @Override<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span>      protected void run() throws IOException {<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2034"></a>
-<span class="sourceLineNo">2035</span><a name="line.2035"></a>
-<span class="sourceLineNo">2036</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2036"></a>
+<span class="sourceLineNo">1862</span>    assert(regionsToMerge.length == 2);<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span><a name="line.1863"></a>
+<span class="sourceLineNo">1864</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span><a name="line.1868"></a>
+<span class="sourceLineNo">1869</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span>      throw new IOException (<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span>    }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span><a name="line.1874"></a>
+<span class="sourceLineNo">1875</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span>      throw new MergeRegionException(<a name="line.1876"></a>
+<span class="sourceLineNo">1877</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span>    }<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span><a name="line.1879"></a>
+<span class="sourceLineNo">1880</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span>      @Override<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span>      protected void run() throws IOException {<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span><a name="line.1888"></a>
+<span class="sourceLineNo">1889</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span>          regionsToMerge, forcible));<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span><a name="line.1891"></a>
+<span class="sourceLineNo">1892</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span>      }<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span><a name="line.1894"></a>
+<span class="sourceLineNo">1895</span>      @Override<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span>      protected String getDescription() {<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span>        return "MergeTableProcedure";<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span>      }<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span>    });<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span>  }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span><a name="line.1901"></a>
+<span class="sourceLineNo">1902</span>  @Override<a name="line.1902"></a>
+<span class="sourceLineNo">1903</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span>      final long nonceGroup, final long nonce)<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span>  throws IOException {<a name="line.1905"></a>
+<span class="sourceLineNo">1906</span>    checkInitialized();<a name="line.1906"></a>
+<span class="sourceLineNo">1907</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>      @Override<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>      protected void run() throws IOException {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        // Execute the operation asynchronously<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>      }<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>      @Override<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>      protected String getDescription() {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span>        return "SplitTableProcedure";<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>      }<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>    });<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  }<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span><a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span>  // Replace with an async implementation from which you can get<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>  // a success/failure result.<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span>  @VisibleForTesting<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    RegionInfo hri;<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    if (regionState != null) {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      hri = regionState.getRegion();<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    } else {<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span>    }<a name="line.1938"></a>
+<span class="sourceLineNo">1939</span><a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>    ServerName dest;<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>        : new ArrayList&lt;&gt;(1);<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>      LOG.info(<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span>              + " because the server is in exclude list");<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>      destServerName = null;<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    }<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span>        "choosing a server at random");<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>      exclude.add(regionState.getServerName());<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span>      if (dest == null) {<a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        return;<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>      }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span>    } else {<a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>      if (dest == null) {<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>        return;<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      }<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>      // TODO: What is this? I don't get it.<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>        // regions on master.<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>          + " because it should not be on master");<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>        return;<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>      }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span>    }<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span><a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>      return;<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>    }<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span><a name="line.1983"></a>
+<span class="sourceLineNo">1984</span>    // Now we can do the move<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span><a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>    try {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      checkInitialized();<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      if (this.cpHost != null) {<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span>      }<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      // closed<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span><a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      try {<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>        // Is this going to work? Will we throw exception on error?<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        future.get();<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        throw new HBaseIOException(e);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>      if (this.cpHost != null) {<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>      }<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>    } catch (IOException ioe) {<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>      if (ioe instanceof HBaseIOException) {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>        throw (HBaseIOException)ioe;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>      }<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      throw new HBaseIOException(ioe);<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>    }<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>  }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>  @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>  public long createTable(<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      final TableDescriptor tableDescriptor,<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>      final byte [][] splitKeys,<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>      final long nonceGroup,<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>      final long nonce) throws IOException {<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span>    checkInitialized();<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span><a name="line.2025"></a>
+<span class="sourceLineNo">2026</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2027"></a>
+<span class="sourceLineNo">2028</span><a name="line.2028"></a>
+<span class="sourceLineNo">2029</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span><a name="line.2031"></a>
+<span class="sourceLineNo">2032</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span>      @Override<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span>      protected void run() throws IOException {<a name="line.2035"></a>
+<span class="sourceLineNo">2036</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2036"></a>
 <span class="sourceLineNo">2037</span><a name="line.2037"></a>
-<span class="sourceLineNo">2038</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span>        //<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2042"></a>
-<span class="sourceLineNo">2043</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span>        submitProcedure(new CreateTableProcedure(<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span>        latch.await();<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span>      }<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span><a name="line.2050"></a>
-<span class="sourceLineNo">2051</span>      @Override<a name="line.2051"></a>
-<span class="sourceLineNo">2052</span>      protected String getDescription() {<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span>        return "CreateTableProcedure";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span>      }<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span>    });<a name="line.2055"></a>
-<span class="sourceLineNo">2056</span>  }<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span><a name="line.2057"></a>
-<span class="sourceLineNo">2058</span>  @Override<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2059"></a>
-<span class="sourceLineNo">2060</span>    if (isStopped()) {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span>      throw new MasterNotRunningException();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span>    }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span>    if (!(tableName.isSystemTable())) {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span>      throw new IllegalArgumentException(<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span>        "Only system table creation can use this createSystemTable API");<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span>    }<a name="line.2068"></a>
-<span class="sourceLineNo">2069</span><a name="line.2069"></a>
-<span class="sourceLineNo">2070</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2070"></a>
+<span class="sourceLineNo">2038</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span><a name="line.2039"></a>
+<span class="sourceLineNo">2040</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2041"></a>
+<span class="sourceLineNo">2042</span>        //<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span>        submitProcedure(new CreateTableProcedure(<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span>        latch.await();<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span>      }<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span><a name="line.2052"></a>
+<span class="sourceLineNo">2053</span>      @Override<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span>      protected String getDescription() {<a name="line.2054"></a>
+<span class="sourceLineNo">2055</span>        return "CreateTableProcedure";<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span>      }<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span>    });<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span>  }<a name="line.2058"></a>
+<span class="sourceLineNo">2059</span><a name="line.2059"></a>
+<span class="sourceLineNo">2060</span>  @Override<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span>    if (isStopped()) {<a name="line.2062"></a>
+<span class="sourceLineNo">2063</span>      throw new MasterNotRunningException();<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span>    }<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span><a name="line.2065"></a>
+<span class="sourceLineNo">2066</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2066"></a>
+<span class="sourceLineNo">2067</span>    if (!(tableName.isSystemTable())) {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span>      throw new IllegalArgumentException(<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span>        "Only system table creation can use this createSystemTable API");<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span>    }<a name="line.2070"></a>
 <span class="sourceLineNo">2071</span><a name="line.2071"></a>
-<span class="sourceLineNo">2072</span>    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2072"></a>
+<span class="sourceLineNo">2072</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2072"></a>
 <span class="sourceLineNo">2073</span><a name="line.2073"></a>
-<span class="sourceLineNo">2074</span>    // This special create table is called locally to master.  Therefore, no RPC means no need<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span>    // to use nonce to detect duplicated RPC call.<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span>    long procId = this.procedureExecutor.submitProcedure(<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span>      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span><a name="line.2078"></a>
-<span class="sourceLineNo">2079</span>    return procId;<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span>  }<a name="line.2080"></a>
-<span class="sourceLineNo">2081</span><a name="line.2081"></a>
-<span class="sourceLineNo">2082</span>  /**<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span>   * Checks whether the table conforms to some sane limits, and configured<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span>   * values (compression, etc) work. Throws an exception if something is wrong.<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span>   * @throws IOException<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span>   */<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span>  private void sanityCheckTableDescriptor(final TableDescriptor htd) throws IOException {<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span>    final String CONF_KEY = "hbase.table.sanity.checks";<a name="line.2088"></a>
-<span class="sourceLineNo">2089</span>    boolean logWarn = false;<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span>    if (!conf.getBoolean(CONF_KEY, true)) {<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span>      logWarn = true;<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span>    }<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span>    String tableVal = htd.getValue(CONF_KEY);<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span>    if (tableVal != null &amp;&amp; !Boolean.valueOf(tableVal)) {<a name="line.2094"></a>
-<span class="sourceLineNo">2095</span>      logWarn = true;<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span>    }<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span><a name="line.2097"></a>
-<span class="sourceLineNo">2098</span>    // check max file size<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span>    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span>    long maxFileSize = htd.getMaxFileSize();<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span>    if (maxFileSize &lt; 0) {<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span>      maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span>    }<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span>    if (maxFileSize &lt; conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span>      String message = "MAX_FILESIZE for table descriptor or "<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span>          + "\"hbase.hregion.max.filesize\" (" + maxFileSize<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span>          + ") is too small, which might cause over splitting into unmanageable "<a name="line.2107"></a>
-<span class="sourceLineNo">2108</span>          + "number of regions.";<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span>    }<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span><a name="line.2111"></a>
-<span class="sourceLineNo">2112</span>    // check flush size<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span>    long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span>    long flushSize = htd.getMemStoreFlushSize();<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span>    if (flushSize &lt; 0) {<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span>      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);<a name="line.2116"></a>
-<span class="sourceLineNo">2117</span>    }<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span>    if (flushSize &lt; conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span>      String message = "MEMSTORE_FLUSHSIZE for table descriptor or "<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span>          + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span>          + " very frequent flushing.";<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2122"></a>
-<span class="sourceLineNo">2123</span>    }<a name="line.2123"></a>
-<span class="sourceLineNo">2124</span><a name="line.2124"></a>
-<span class="sourceLineNo">2125</span>    // check that coprocessors and other specified plugin classes can be loaded<a name="line.2125"></a>
-<span class="sourceLineNo">2126</span>    try {<a name="line.2126"></a>
-<span class="sourceLineNo">2127</span>      checkClassLoading(conf, htd);<a name="line.2127"></a>
-<span class="sourceLineNo">2128</span>    } catch (Exception ex) {<a name="line.2128"></a>
-<span class="sourceLineNo">2129</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);<a name="line.2129"></a>
-<span class="sourceLineNo">2130</span>    }<a name="line.2130"></a>
-<span class="sourceLineNo">2131</span><a name="line.2131"></a>
-<span class="sourceLineNo">2132</span>    // check compression can be loaded<a name="line.2132"></a>
-<span class="sourceLineNo">2133</span>    try {<a name="line.2133"></a>
-<span class="sourceLineNo">2134</span>      checkCompression(htd);<a name="line.2134"></a>
-<span class="sourceLineNo">2135</span>    } catch (IOException e) {<a name="line.2135"></a>
-<span class="sourceLineNo">2136</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2136"></a>
-<span class="sourceLineNo">2137</span>    }<a name="line.2137"></a>
-<span class="sourceLineNo">2138</span><a name="line.2138"></a>
-<span class="sourceLineNo">2139</span>    // check encryption can be loaded<a name="line.2139"></a>
-<span class="sourceLineNo">2140</span>    try {<a name="line.2140"></a>
-<span class="sourceLineNo">2141</span>      checkEncryption(conf, htd);<a name="line.2141"></a>
-<span class="sourceLineNo">2142</span>    } catch (IOException e) {<a name="line.2142"></a>
-<span class="sourceLineNo">2143</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2143"></a>
-<span class="sourceLineNo">2144</span>    }<a name="line.2144"></a>
-<span class="sourceLineNo">2145</span>    // Verify compaction policy<a name="line.2145"></a>
-<span class="sourceLineNo">2146</span>    try{<a name="line.2146"></a>
-<span class="sourceLineNo">2147</span>      checkCompactionPolicy(conf, htd);<a name="line.2147"></a>
-<span class="sourceLineNo">2148</span>    } catch(IOException e){<a name="line.2148"></a>
-<span class="sourceLineNo">2149</span>      warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);<a name="line.2149"></a>
-<span class="sourceLineNo">2150</span>    }<a name="line.2150"></a>
-<span class="sourceLineNo">2151</span>    // check that we have at least 1 CF<a name="line.2151"></a>
-<span class="sourceLineNo">2152</span>    if (htd.getColumnFamilyCount() == 0) {<a name="line.2152"></a>
-<span class="sourceLineNo">2153</span>      String message = "Table should have at least one column family.";<a name="line.2153"></a>
-<span class="sourceLineNo">2154</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2154"></a>
-<span class="sourceLineNo">2155</span>    }<a name="line.2155"></a>
-<span class="sourceLineNo">2156</span><a name="line.2156"></a>
-<span class="sourceLineNo">2157</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2157"></a>
-<span class="sourceLineNo">2158</span>      if (hcd.getTimeToLive() &lt;= 0) {<a name="line.2158"></a>
-<span class="sourceLineNo">2159</span>        String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";<a name="line.2159"></a>
-<span class="sourceLineNo">2160</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2160"></a>
-<span class="sourceLineNo">2161</span>      }<a name="line.2161"></a>
-<span class="sourceLineNo">2162</span><a name="line.2162"></a>
-<span class="sourceLineNo">2163</span>      // check blockSize<a name="line.2163"></a>
-<span class="sourceLineNo">2164</span>      if (hcd.getBlocksize() &lt; 1024 || hcd.getBlocksize() &gt; 16 * 1024 * 1024) {<a name="line.2164"></a>
-<span class="sourceLineNo">2165</span>        String message = "Block size for column family " + hcd.getNameAsString()<a name="line.2165"></a>
-<span class="sourceLineNo">2166</span>            + "  must be between 1K and 16MB.";<a name="line.2166"></a>
-<span class="sourceLineNo">2167</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2167"></a>
-<span class="sourceLineNo">2168</span>      }<a name="line.2168"></a>
-<span class="sourceLineNo">2169</span><a name="line.2169"></a>
-<span class="sourceLineNo">2170</span>      // check versions<a name="line.2170"></a>
-<span class="sourceLineNo">2171</span>      if (hcd.getMinVersions() &lt; 0) {<a name="line.2171"></a>
-<span class="sourceLineNo">2172</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2172"></a>
-<span class="sourceLineNo">2173</span>          + "  must be positive.";<a name="line.2173"></a>
-<span class="sourceLineNo">2174</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2174"></a>
-<span class="sourceLineNo">2175</span>      }<a name="line.2175"></a>
-<span class="sourceLineNo">2176</span>      // max versions already being checked<a name="line.2176"></a>
-<span class="sourceLineNo">2177</span><a name="line.2177"></a>
-<span class="sourceLineNo">2178</span>      // HBASE-13776 Setting illegal versions for ColumnFamilyDescriptor<a name="line.2178"></a>
-<span class="sourceLineNo">2179</span>      //  does not throw IllegalArgumentException<a name="line.2179"></a>
-<span class="sourceLineNo">2180</span>      // check minVersions &lt;= maxVerions<a name="line.2180"></a>
-<span class="sourceLineNo">2181</span>      if (hcd.getMinVersions() &gt; hcd.getMaxVersions()) {<a name="line.2181"></a>
-<span class="sourceLineNo">2182</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2182"></a>
-<span class="sourceLineNo">2183</span>            + " must be less than the Max versions.";<a name="line.2183"></a>
-<span class="sourceLineNo">2184</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2184"></a>
-<span class="sourceLineNo">2185</span>      }<a name="line.2185"></a>
-<span class="sourceLineNo">2186</span><a name="line.2186"></a>
-<span class="sourceLineNo">2187</span>      // check replication scope<a name="line.2187"></a>
-<span class="sourceLineNo">2188</span>      checkReplicationScope(hcd);<a name="line.2188"></a>
-<span class="sourceLineNo">2189</span>      // check bloom filter type<a name="line.2189"></a>
-<span class="sourceLineNo">2190</span>      checkBloomFilterType(hcd);<a name="line.2190"></a>
-<span class="sourceLineNo">2191</span><a name="line.2191"></a>
-<span class="sourceLineNo">2192</span>      // check data replication factor, it can be 0(default value) when user has not explicitly<a name="line.2192"></a>
-<span class="sourceLineNo">2193</span>      // set the value, in this case we use default replication factor set in the file system.<a name="line.2193"></a>
-<span class="sourceLineNo">2194</span>      if (hcd.getDFSReplication() &lt; 0) {<a name="line.2194"></a>
-<span class="sourceLineNo">2195</span>        String message = "HFile Replication for column family " + hcd.getNameAsString()<a name="line.2195"></a>
-<span class="sourceLineNo">2196</span>            + "  must be greater than zero.";<a name="line.2196"></a>
-<span class="sourceLineNo">2197</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2197"></a>
-<span class="sourceLineNo">2198</span>      }<a name="line.2198"></a>
-<span class="sourceLineNo">2199</span><a name="line.2199"></a>
-<span class="sourceLineNo">2200</span>      // TODO: should we check coprocessors and encryption ?<a name="line.2200"></a>
-<span class="sourceLineNo">2201</span>    }<a name="line.2201"></a>
-<span class="sourceLineNo">2202</span>  }<a name="line.2202"></a>
-<span class="sourceLineNo">2203</span><a name="line.2203"></a>
-<span class="sourceLineNo">2204</span>  private void checkReplicationScope(ColumnFamilyDescriptor hcd) throws IOException{<a name="line.2204"></a>
-<span class="sourceLineNo">2205</span>    // check replication scope<a name="line.2205"></a>
-<span class="sourceLineNo">2206</span>    WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(hcd.getScope());<a name="line.2206"></a>
-<span class="sourceLineNo">2207</span>    if (scop == null) {<a name="line.2207"></a>
-<span class="sourceLineNo">2208</span>      String message = "Replication scope for column family "<a name="line.2208"></a>
-<span class="sourceLineNo">2209</span>          + hcd.getNameAsString() + " is " + hcd.getScope() + " which is invalid.";<a name="line.2209"></a>
-<span class="sourceLineNo">2210</span><a name="line.2210"></a>
-<span class="sourceLineNo">2211</span>      LOG.error(message);<a name="line.2211"></a>
-<span class="sourceLineNo">2212</span>      throw new DoNotRetryIOException(message);<a name="line.2212"></a>
-<span class="sourceLineNo">2213</span>    }<a name="line.2213"></a>
-<span class="sourceLineNo">2214</span>  }<a name="line.2214"></a>
-<span class="sourceLineNo">2215</span><a name="line.2215"></a>
-<span class="sourceLineNo">2216</span>  private void checkCompactionPolicy(Configuration conf, TableDescriptor htd)<a name="line.2216"></a>
-<span class="sourceLineNo">2217</span>      throws IOException {<a name="line.2217"></a>
-<span class="sourceLineNo">2218</span>    // FIFO compaction has some requirements<a name="line.2218"></a>
-<span class="sourceLineNo">2219</span>    // Actually FCP ignores periodic major compactions<a name="line.2219"></a>
-<span class="sourceLineNo">2220</span>    String className = htd.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2220"></a>
-<span class="sourceLineNo">2221</span>    if (className == null) {<a name="line.2221"></a>
-<span class="sourceLineNo">2222</span>      className =<a name="line.2222"></a>
-<span class="sourceLineNo">2223</span>          conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,<a name="line.2223"></a>
-<span class="sourceLineNo">2224</span>            ExploringCompactionPolicy.class.getName());<a name="line.2224"></a>
-<span class="sourceLineNo">2225</span>    }<a name="line.2225"></a>
-<span class="sourceLineNo">2226</span><a name="line.2226"></a>
-<span class="sourceLineNo">2227</span>    int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;<a name="line.2227"></a>
-<span class="sourceLineNo">2228</span>    String sv = htd.getValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2228"></a>
-<span class="sourceLineNo">2229</span>    if (sv != null) {<a name="line.2229"></a>
-<span class="sourceLineNo">2230</span>      blockingFileCount = Integer.parseInt(sv);<a name="line.2230"></a>
-<span class="sourceLineNo">2231</span>    } else {<a name="line.2231"></a>
-<span class="sourceLineNo">2232</span>      blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);<a name="line.2232"></a>
-<span class="sourceLineNo">2233</span>    }<a name="line.2233"></a>
-<span class="sourceLineNo">2234</span><a name="line.2234"></a>
-<span class="sourceLineNo">2235</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2235"></a>
-<span class="sourceLineNo">2236</span>      String compactionPolicy =<a name="line.2236"></a>
-<span class="sourceLineNo">2237</span>          hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2237"></a>
-<span class="sourceLineNo">2238</span>      if (compactionPolicy == null) {<a name="line.2238"></a>
-<span class="sourceLineNo">2239</span>        compactionPolicy = className;<a name="line.2239"></a>
-<span class="sourceLineNo">2240</span>      }<a name="line.2240"></a>
-<span class="sourceLineNo">2241</span>      if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {<a name="line.2241"></a>
-<span class="sourceLineNo">2242</span>        continue;<a name="line.2242"></a>
-<span class="sourceLineNo">2243</span>      }<a name="line.2243"></a>
-<span class="sourceLineNo">2244</span>      // FIFOCompaction<a name="line.2244"></a>
-<span class="sourceLineNo">2245</span>      String message = null;<a name="line.2245"></a>
-<span class="sourceLineNo">2246</span><a name="line.2246"></a>
-<span class="sourceLineNo">2247</span>      // 1. Check TTL<a name="line.2247"></a>
-<span class="sourceLineNo">2248</span>      if (hcd.getTimeToLive() == ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {<a name="line.2248"></a>
-<span class="sourceLineNo">2249</span>        message = "Default TTL is not supported for FIFO compaction";<a name="line.2249"></a>
-<span class="sourceLineNo">2250</span>        throw new IOException(message);<a name="line.2250"></a>
-<span class="sourceLineNo">2251</span>      }<a name="line.2251"></a>
-<span class="sourceLineNo">2252</span><a name="line.2252"></a>
-<span class="sourceLineNo">2253</span>      // 2. Check min versions<a name="line.2253"></a>
-<span class="sourceLineNo">2254</span>      if (hcd.getMinVersions() &gt; 0) {<a name="line.2254"></a>
-<span class="sourceLineNo">2255</span>        message = "MIN_VERSION &gt; 0 is not supported for FIFO compaction";<a name="line.2255"></a>
-<span class="sourceLineNo">2256</span>        throw new IOException(message);<a name="line.2256"></a>
-<span class="sourceLineNo">2257</span>      }<a name="line.2257"></a>
-<span class="sourceLineNo">2258</span><a name="line.2258"></a>
-<span class="sourceLineNo">2259</span>      // 3. blocking file count<a name="line.2259"></a>
-<span class="sourceLineNo">2260</span>      sv = hcd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2260"></a>
-<span class="sourceLineNo">2261</span>      if (sv != null) {<a name="line.2261"></a>
-<span class="sourceLineNo">2262</span>        blockingFileCount = Integer.parseInt(sv);<a name="line.2262"></a>
-<span class="sourceLineNo">2263</span>      }<a name="line.2263"></a>
-<span class="sourceLineNo">2264</span>      if (blockingFileCount &lt; 1000) {<a name="line.2264"></a>
-<span class="sourceLineNo">2265</span>        message =<a name="line.2265"></a>
-<span class="sourceLineNo">2266</span>            "Blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount<a name="line.2266"></a>
-<span class="sourceLineNo">2267</span>                + " is below recommended minimum of 1000 for column family "+ hcd.getNameAsString();<a name="line.2267"></a>
-<span class="sourceLineNo">2268</span>        throw new IOException(message);<a name="line.2268"></a>
-<span class="sourceLineNo">2269</span>      }<a name="line.2269"></a>
-<span class="sourceLineNo">2270</span>    }<a name="line.2270"></a>
-<span class="sourceLineNo">2271</span>  }<a name="line.2271"></a>
-<span class="sourceLineNo">2272</span><a name="line.2272"></a>
-<span class="sourceLineNo">2273</span>  private static void checkBloomFilterType(ColumnFamilyDescriptor cfd)<a name="line.2273"></a>
-<span class="sourceLineNo">2274</span>      throws IOException {<a name="line.2274"></a>
-<span class="sourceLineNo">2275</span>    Configuration conf = new CompoundConfiguration().addStringMap(cfd.getConfiguration());<a name="line.2275"></a>
-<span class="sourceLineNo">2276</span>    try {<a name="line.2276"></a>
-<span class="sourceLineNo">2277</span>      BloomFilterUtil.getBloomFilterParam(cfd.getBloomFilterType(), conf);<a name="line.2277"></a>
-<span class="sourceLineNo">2278</span>    } catch (IllegalArgumentException e) {<a name="line.2278"></a>
-<span class="sourceLineNo">2279</span>      throw new DoNotRetryIOException("Failed to get bloom filter param", e);<a name="line.2279"></a>
-<span class="sourceLineNo">2280</span>    }<a name="line.2280"></a>
-<span class="sourceLineNo">2281</span>  }<a name="line.2281"></a>
-<span class="sourceLineNo">2282</span><a name="line.2282"></a>
-<span class="sourceLineNo">2283</span>  // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.<a name="line.2283"></a>
-<span class="sourceLineNo">2284</span>  private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,<a name="line.2284"></a>
-<span class="sourceLineNo">2285</span>      String message, Exception cause) throws IOException {<a name="line.2285"></a>
-<span class="sourceLineNo">2286</span>    if (!logWarn) {<a name="line.2286"></a>
-<span class="sourceLineNo">2287</span>      throw new DoNotRetryIOException(message + " Set " + confKey +<a name="line.2287"></a>
-<span class="sourceLineNo">2288</span>          " to false at conf or table descriptor if you want to bypass sanity checks", cause);<a name="line.2288"></a>
-<span class="sourceLineNo">2289</span>    }<a name="line.2289"></a>
-<span class="sourceLineNo">2290</span>    LOG.warn(message);<a name="line.2290"></a>
-<span class="sourceLineNo">2291</span>  }<a name="line.2291"></a>
-<span class="sourceLineNo">2292</span><a name="line.2292"></a>
-<span class="sourceLineNo">2293</span>  private void startActiveMasterManager(int infoPort) throws KeeperException {<a name="line.2293"></a>
-<span class="sourceLineNo">2294</span>    String backupZNode = ZNodePaths.joinZNode(<a name="line.2294"></a>
-<span class="sourceLineNo">2295</span>      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());<a name="line.2295"></a>
-<span class="sourceLineNo">2296</span>    /*<a name="line.2296"></a>
-<span class="sourceLineNo">2297</span>    * Add a ZNode for ourselves in the backup master directory since we<a name="line.2297"></a>
-<span class="sourceLineNo">2298</span>    * may not become the active master. If so, we want the actual active<a name="line.2298"></a>
-<span class="sourceLineNo">2299</span>    * master to know we are backup masters, so that it won't assign<a name="line.2299"></a>
-<span class="sourceLineNo">2300</span>    * regions to us if so configured.<a name="line.2300"></a>
-<span class="sourceLineNo">2301</span>    *<a name="line.2301"></a>
-<span class="sourceLineNo">2302</span>    * If we become the active master later, ActiveMasterManager will delete<a name="line.2302"></a>
-<span class="sourceLineNo">2303</span>    * this node explicitly.  If we crash before then, ZooKeeper will delete<a name="line.2303"></a>
-<span class="sourceLineNo">2304</span>    * this node for us since it is ephemeral.<a name="line.2304"></a>
-<span class="sourceLineNo">2305</span>    */<a name="line.2305"></a>
-<span class="sourceLineNo">2306</span>    LOG.info("Adding backup master ZNode " + backupZNode);<a name="line.2306"></a>
-<span class="sourceLineNo">2307</span>    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {<a name="line.2307"></a>
-<span class="sourceLineNo">2308</span>      LOG.warn("Failed create of " + backupZNode + " by " + serverName);<a name="line.2308"></a>
-<span class="sourceLineNo">2309</span>    }<a name="line.2309"></a>
-<span class="sourceLineNo">2310</span>    this.activeMasterManager.setInfoPort(infoPort);<a name="line.2310"></a>
-<span class="sourceLineNo">2311</span>    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);<a name="line.2311"></a>
-<span class="sourceLineNo">2312</span>    // If we're a backup master, stall until a primary to write this address<a name="line.2312"></a>
-<span class="sourceLineNo">2313</span>    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {<a name="line.2313"></a>
-<span class="sourceLineNo">2314</span>      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");<a name="line.2314"></a>
-<span class="sourceLineNo">2315</span>      // This will only be a minute or so while the cluster starts up,<a name="line.2315"></a>
-<span class="sourceLineNo">2316</span>      // so don't worry about setting watches on the parent znode<a name="line.2316"></a>
-<span class="sourceLineNo">2317</span>      while (!activeMasterManager.hasActiveMaster()) {<a name="line.2317"></a>
-<span class="sourceLineNo">2318</span>        LOG.debug("Waiting for master address and cluster state znode to be written.");<a name="line.2318"></a>
-<span class="sourceLineNo">2319</span>        Threads.sleep(timeout);<a name="line.2319"></a>
-<span class="sourceLineNo">2320</span>      }<a name="line.2320"></a>
-<span class="sourceLineNo">2321</span>    }<a name="line.2321"></a>
-<span class="sourceLineNo">2322</span>    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");<a name="line.2322"></a>
-<span class="sourceLineNo">2323</span>    status.setDescription("Master startup");<a name="line.2323"></a>
-<span class="sourceLineNo">2324</span>    try {<a name="line.2324"></a>
-<span class="sourceLineNo">2325</span>      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {<a name="line.2325"></a>
-<span class="sourceLineNo">2326</span>        finishActiveMasterInitialization(status);<a name="line.2326"></a>
-<span class="sourceLineNo">2327</span>      }<a name="line.2327"></a>
-<span class="sourceLineNo">2328</span>    } catch (Throwable t) {<a name="line.2328"></a>
-<span class="sourceLineNo">2329</span>      status.setStatus("Failed to become active: " + t.getMessage());<a name="line.2329"></a>
-<span class="sourceLineNo">2330</span>      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);<a name="line.2330"></a>
-<span class="sourceLineNo">2331</span>      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility<a name="line.2331"></a>
-<span class="sourceLineNo">2332</span>      if (t instanceof NoClassDefFoundError &amp;&amp; t.getMessage().<a name="line.2332"></a>
-<span class="sourceLineNo">2333</span>          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {<a name="line.2333"></a>
-<span class="sourceLineNo">2334</span>        // improved error message for this special case<a name="line.2334"></a>
-<span class="sourceLineNo">2335</span>        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +<a name="line.2335"></a>
-<span class="sourceLineNo">2336</span>          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +<a name="line.2336"></a>
-<span class="sourceLineNo">2337</span>          " or change your hadoop jars to start properly", t);<a name="line.2337"></a>
-<span class="sourceLineNo">2338</span>      } else {<a name="line.2338"></a>
-<span class="sourceLineNo">2339</span>        abort("Unhandled exception. Starting shutdown.", t);<a name="line.2339"></a>
-<span class="sourceLineNo">2340</span>      }<a name="line.2340"></a>
-<span class="sourceLineNo">2341</span>    } finally {<a name="line.2341"></a>
-<span class="sourceLineNo">2342</span>      status.cleanup();<a name="line.2342"></a>
-<span class="sourceLineNo">2343</span>    }<a name="line.2343"></a>
-<span class="sourceLineNo">2344</span>  }<a name="line.2344"></a>
-<span class="sourceLineNo">2345</span><a name="line.2345"></a>
-<span class="sourceLineNo">2346</span>  private void checkCompression(final TableDescriptor htd)<a name="line.2346"></a>
-<span class="sourceLineNo">2347</span>  throws IOException {<a name="line.2347"></a>
-<span class="sourceLineNo">2348</span>    if (!this.masterCheckCompression) return;<a name="line.2348"></a>
-<span class="sourceLineNo">2349</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2349"></a>
-<span class="sourceLineNo">2350</span>      checkCompression(hcd);<a name="line.2350"></a>
-<span class="sourceLineNo">2351</span>    }<a name="line.2351"></a>
-<span class="sourceLineNo">2352</span>  }<a name="line.2352"></a>
-<span class="sourceLineNo">2353</span><a name="line.2353"></a>
-<span class="sourceLineNo">2354</span>  private void checkCompression(final ColumnFamilyDescriptor hcd)<a name="line.2354"></a>
-<span class="sourceLineNo">2355</span>  throws IOException {<a name="line.2355"></a>
-<span class="sourceLineNo">2356</span>    if (!this.masterCheckCompression) return;<a name="line.2356"></a>
-<span class="sourceLineNo">2357</span>    CompressionTest.testCompression(hcd.getCompressionType());<a name="line.2357"></a>
-<span class="sourceLineNo">2358</span>    CompressionTest.testCompression(hcd.getCompactionCompressionType());<a name="line.2358"></a>
-<span class="sourceLineNo">2359</span>  }<a name="line.2359"></a>
-<span class="sourceLineNo">2360</span>

<TRUNCATED>

[49/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/HMaster.html b/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
index 6f4077e..13b1730 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/HMaster.html
@@ -2943,7 +2943,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClientIdAuditPrefix</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1837">getClientIdAuditPrefix</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1839">getClientIdAuditPrefix</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getClientIdAuditPrefix--">getClientIdAuditPrefix</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -2958,7 +2958,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>setCatalogJanitorEnabled</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1848">setCatalogJanitorEnabled</a>(boolean&nbsp;b)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1850">setCatalogJanitorEnabled</a>(boolean&nbsp;b)</pre>
 <div class="block">Switch for the background CatalogJanitor thread.
  Used for testing.  The thread will continue to run.  It will just be a noop
  if disabled.</div>
@@ -2974,7 +2974,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>mergeRegions</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1853">mergeRegions</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>[]&nbsp;regionsToMerge,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1855">mergeRegions</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>[]&nbsp;regionsToMerge,
                          boolean&nbsp;forcible,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3002,7 +3002,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>splitRegion</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1901">splitRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionInfo,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1903">splitRegion</a>(<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>&nbsp;regionInfo,
                         byte[]&nbsp;splitRow,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -3030,7 +3030,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>move</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1927">move</a>(byte[]&nbsp;encodedRegionName,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.1929">move</a>(byte[]&nbsp;encodedRegionName,
                  byte[]&nbsp;destServerName)
           throws <a href="../../../../../org/apache/hadoop/hbase/HBaseIOException.html" title="class in org.apache.hadoop.hbase">HBaseIOException</a></pre>
 <dl>
@@ -3045,7 +3045,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2017">createTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2019">createTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor,
                         byte[][]&nbsp;splitKeys,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -3070,7 +3070,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createSystemTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2059">createSystemTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor)
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2061">createSystemTable</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;tableDescriptor)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#createSystemTable-org.apache.hadoop.hbase.client.TableDescriptor-">MasterServices</a></code></span></div>
 <div class="block">Create a system table using the given table definition.</div>
@@ -3091,7 +3091,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>sanityCheckTableDescriptor</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2087">sanityCheckTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2089">sanityCheckTableDescriptor</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Checks whether the table conforms to some sane limits, and configured
  values (compression, etc) work. Throws an exception if something is wrong.</div>
@@ -3107,7 +3107,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkReplicationScope</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2204">checkReplicationScope</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2206">checkReplicationScope</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3121,7 +3121,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompactionPolicy</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2216">checkCompactionPolicy</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2218">checkCompactionPolicy</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                                    <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3136,7 +3136,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkBloomFilterType</h4>
-<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2273">checkBloomFilterType</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;cfd)
+<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2275">checkBloomFilterType</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;cfd)
                                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3150,7 +3150,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>warnOrThrowExceptionForFailure</h4>
-<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2284">warnOrThrowExceptionForFailure</a>(boolean&nbsp;logWarn,
+<pre>private static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2286">warnOrThrowExceptionForFailure</a>(boolean&nbsp;logWarn,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;confKey,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;message,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a>&nbsp;cause)
@@ -3167,7 +3167,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>startActiveMasterManager</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2293">startActiveMasterManager</a>(int&nbsp;infoPort)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2295">startActiveMasterManager</a>(int&nbsp;infoPort)
                                throws org.apache.zookeeper.KeeperException</pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3181,7 +3181,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompression</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2346">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2348">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3195,7 +3195,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkCompression</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2354">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2356">checkCompression</a>(<a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3209,7 +3209,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkEncryption</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2361">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2363">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                              <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                       throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3224,7 +3224,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkEncryption</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2369">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2371">checkEncryption</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                              <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;hcd)
                       throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3239,7 +3239,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkClassLoading</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2375">checkClassLoading</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2377">checkClassLoading</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                                <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;htd)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -3254,7 +3254,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isCatalogTable</h4>
-<pre>private static&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2381">isCatalogTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)</pre>
+<pre>private static&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2383">isCatalogTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)</pre>
 </li>
 </ul>
 <a name="deleteTable-org.apache.hadoop.hbase.TableName-long-long-">
@@ -3263,7 +3263,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2386">deleteTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2388">deleteTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3285,7 +3285,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>truncateTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2420">truncateTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2422">truncateTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                           boolean&nbsp;preserveSplits,
                           long&nbsp;nonceGroup,
                           long&nbsp;nonce)
@@ -3309,7 +3309,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>addColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2450">addColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2452">addColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                       <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;column,
                       long&nbsp;nonceGroup,
                       long&nbsp;nonce)
@@ -3333,7 +3333,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2478">modifyColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2480">modifyColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          <a href="../../../../../org/apache/hadoop/hbase/client/ColumnFamilyDescriptor.html" title="interface in org.apache.hadoop.hbase.client">ColumnFamilyDescriptor</a>&nbsp;descriptor,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3357,7 +3357,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteColumn</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2498">deleteColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2500">deleteColumn</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          byte[]&nbsp;columnName,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3381,7 +3381,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>enableTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2523">enableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2525">enableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3403,7 +3403,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>disableTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2576">disableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2578">disableTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -3425,7 +3425,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getTableRegionForRow</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2618">getTableRegionForRow</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre><a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/RegionInfo.html" title="interface in org.apache.hadoop.hbase.client">RegionInfo</a>,<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2620">getTableRegionForRow</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                                                  byte[]&nbsp;rowKey)
                                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Return the region and current deployment for the region containing
@@ -3444,7 +3444,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyTable</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2644">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>private&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2646">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                          <a href="../../../../../org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html" title="interface in org.apache.hadoop.hbase.master">HMaster.TableDescriptorGetter</a>&nbsp;newDescriptorGetter,
                          long&nbsp;nonceGroup,
                          long&nbsp;nonce)
@@ -3461,7 +3461,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyTable</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2682">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2684">modifyTable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                         <a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&nbsp;newDescriptor,
                         long&nbsp;nonceGroup,
                         long&nbsp;nonce)
@@ -3485,7 +3485,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>restoreSnapshot</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2694">restoreSnapshot</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription&nbsp;snapshotDesc,
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2696">restoreSnapshot</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription&nbsp;snapshotDesc,
                             long&nbsp;nonceGroup,
                             long&nbsp;nonce,
                             boolean&nbsp;restoreAcl)
@@ -3502,7 +3502,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkTableExists</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2718">checkTableExists</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2720">checkTableExists</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                               <a href="../../../../../org/apache/hadoop/hbase/TableNotFoundException.html" title="class in org.apache.hadoop.hbase">TableNotFoundException</a></pre>
 <dl>
@@ -3518,7 +3518,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkTableModifiable</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2726">checkTableModifiable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2728">checkTableModifiable</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
                                  <a href="../../../../../org/apache/hadoop/hbase/TableNotFoundException.html" title="class in org.apache.hadoop.hbase">TableNotFoundException</a>,
                                  <a href="../../../../../org/apache/hadoop/hbase/TableNotDisabledException.html" title="class in org.apache.hadoop.hbase">TableNotDisabledException</a></pre>
@@ -3542,7 +3542,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetricsWithoutCoprocessor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2738">getClusterMetricsWithoutCoprocessor</a>()
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2740">getClusterMetricsWithoutCoprocessor</a>()
                                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3556,7 +3556,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetricsWithoutCoprocessor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2742">getClusterMetricsWithoutCoprocessor</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2744">getClusterMetricsWithoutCoprocessor</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
                                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3570,7 +3570,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetrics</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2804">getClusterMetrics</a>()
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2806">getClusterMetrics</a>()
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
@@ -3586,7 +3586,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterMetrics</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2808">getClusterMetrics</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2810">getClusterMetrics</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/EnumSet.html?is-external=true" title="class or interface in java.util">EnumSet</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase">ClusterMetrics.Option</a>&gt;&nbsp;options)
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3600,7 +3600,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getBackupMasters</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2819">getBackupMasters</a>()
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2821">getBackupMasters</a>()
                                    throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/InterruptedIOException.html?is-external=true" title="class or interface in java.io">InterruptedIOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3614,7 +3614,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getLoadedCoprocessors</h4>
-<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2872">getLoadedCoprocessors</a>()</pre>
+<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2874">getLoadedCoprocessors</a>()</pre>
 <div class="block">The set of loaded coprocessors is stored in a static set. Since it's
  statically allocated, it does not require that HMaster's cpHost be
  initialized prior to accessing it.</div>
@@ -3630,7 +3630,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterStartTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2879">getMasterStartTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2881">getMasterStartTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster was started.</dd>
@@ -3643,7 +3643,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterActiveTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2886">getMasterActiveTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2888">getMasterActiveTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster became the active master.</dd>
@@ -3656,7 +3656,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterFinishedInitializationTime</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2893">getMasterFinishedInitializationTime</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2895">getMasterFinishedInitializationTime</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>timestamp in millis when HMaster finished becoming the active master</dd>
@@ -3669,7 +3669,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getNumWALFiles</h4>
-<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2897">getNumWALFiles</a>()</pre>
+<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2899">getNumWALFiles</a>()</pre>
 </li>
 </ul>
 <a name="getWalProcedureStore--">
@@ -3678,7 +3678,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getWalProcedureStore</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2901">getWalProcedureStore</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureStore</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2903">getWalProcedureStore</a>()</pre>
 </li>
 </ul>
 <a name="getRegionServerInfoPort-org.apache.hadoop.hbase.ServerName-">
@@ -3687,7 +3687,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerInfoPort</h4>
-<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2905">getRegionServerInfoPort</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
+<pre>public&nbsp;int&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2907">getRegionServerInfoPort</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
 </li>
 </ul>
 <a name="getRegionServerVersion-org.apache.hadoop.hbase.ServerName-">
@@ -3696,7 +3696,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerVersion</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2912">getRegionServerVersion</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2914">getRegionServerVersion</a>(<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;sn)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getRegionServerVersion-org.apache.hadoop.hbase.ServerName-">getRegionServerVersion</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3709,7 +3709,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkIfShouldMoveSystemRegionAsync</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2919">checkIfShouldMoveSystemRegionAsync</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2921">checkIfShouldMoveSystemRegionAsync</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#checkIfShouldMoveSystemRegionAsync--">MasterServices</a></code></span></div>
 <div class="block">Called when a new RegionServer is added to the cluster.
  Checks if new server has a newer version than any existing server and will move system tables
@@ -3726,7 +3726,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterCoprocessors</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2926">getMasterCoprocessors</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2928">getMasterCoprocessors</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>array of coprocessor SimpleNames.</dd>
@@ -3739,7 +3739,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>abort</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2932">abort</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;reason,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2934">abort</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;reason,
                   <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a>&nbsp;cause)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegionServer.html#abort-java.lang.String-java.lang.Throwable-">HRegionServer</a></code></span></div>
 <div class="block">Cause the server to exit without closing the regions it is serving, the log
@@ -3762,7 +3762,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getZooKeeper</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2956">getZooKeeper</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/zookeeper/ZKWatcher.html" title="class in org.apache.hadoop.hbase.zookeeper">ZKWatcher</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2958">getZooKeeper</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Server.html#getZooKeeper--">Server</a></code></span></div>
 <div class="block">Gets the ZooKeeper instance for this server.</div>
 <dl>
@@ -3779,7 +3779,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterCoprocessorHost</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterCoprocessorHost.html" title="class in org.apache.hadoop.hbase.master">MasterCoprocessorHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2961">getMasterCoprocessorHost</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/MasterCoprocessorHost.html" title="class in org.apache.hadoop.hbase.master">MasterCoprocessorHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2963">getMasterCoprocessorHost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterCoprocessorHost--">getMasterCoprocessorHost</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3794,7 +3794,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterQuotaManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/MasterQuotaManager.html" title="class in org.apache.hadoop.hbase.quotas">MasterQuotaManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2966">getMasterQuotaManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/quotas/MasterQuotaManager.html" title="class in org.apache.hadoop.hbase.quotas">MasterQuotaManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2968">getMasterQuotaManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterQuotaManager--">getMasterQuotaManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3809,7 +3809,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterProcedureExecutor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.html" title="class in org.apache.hadoop.hbase.master.procedure">MasterProcedureEnv</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2971">getMasterProcedureExecutor</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.html" title="class in org.apache.hadoop.hbase.master.procedure">MasterProcedureEnv</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2973">getMasterProcedureExecutor</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterProcedureExecutor--">getMasterProcedureExecutor</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3824,7 +3824,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getServerName</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2976">getServerName</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/ServerName.html" title="class in org.apache.hadoop.hbase">ServerName</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2978">getServerName</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/Server.html#getServerName--">getServerName</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Server.html" title="interface in org.apache.hadoop.hbase">Server</a></code></dd>
@@ -3841,7 +3841,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getAssignmentManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/assignment/AssignmentManager.html" title="class in org.apache.hadoop.hbase.master.assignment">AssignmentManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2981">getAssignmentManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/assignment/AssignmentManager.html" title="class in org.apache.hadoop.hbase.master.assignment">AssignmentManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2983">getAssignmentManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getAssignmentManager--">getAssignmentManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3856,7 +3856,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getCatalogJanitor</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/CatalogJanitor.html" title="class in org.apache.hadoop.hbase.master">CatalogJanitor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2986">getCatalogJanitor</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/CatalogJanitor.html" title="class in org.apache.hadoop.hbase.master">CatalogJanitor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2988">getCatalogJanitor</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getCatalogJanitor--">getCatalogJanitor</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -3871,7 +3871,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionServerFatalLogBuffer</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.html" title="class in org.apache.hadoop.hbase.monitoring">MemoryBoundedLogMessageBuffer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2990">getRegionServerFatalLogBuffer</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/monitoring/MemoryBoundedLogMessageBuffer.html" title="class in org.apache.hadoop.hbase.monitoring">MemoryBoundedLogMessageBuffer</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2992">getRegionServerFatalLogBuffer</a>()</pre>
 </li>
 </ul>
 <a name="shutdown--">
@@ -3880,7 +3880,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>shutdown</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.2998">shutdown</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3000">shutdown</a>()
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Shutdown the cluster.
  Master runs a coordinated stop of all RegionServers and then itself.</div>
@@ -3896,7 +3896,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stopMaster</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3030">stopMaster</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3032">stopMaster</a>()
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3910,7 +3910,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>stop</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3038">stop</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;msg)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3040">stop</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;msg)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/Stoppable.html#stop-java.lang.String-">Stoppable</a></code></span></div>
 <div class="block">Stop this service.
  Implementers should favor logging errors over throwing RuntimeExceptions.</div>
@@ -3930,7 +3930,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkServiceStarted</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3048">checkServiceStarted</a>()
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3050">checkServiceStarted</a>()
                             throws <a href="../../../../../org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.html" title="class in org.apache.hadoop.hbase.ipc">ServerNotRunningYetException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -3944,7 +3944,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>checkInitialized</h4>
-<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3060">checkInitialized</a>()
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3062">checkInitialized</a>()
                throws <a href="../../../../../org/apache/hadoop/hbase/PleaseHoldException.html" title="class in org.apache.hadoop.hbase">PleaseHoldException</a>,
                       <a href="../../../../../org/apache/hadoop/hbase/ipc/ServerNotRunningYetException.html" title="class in org.apache.hadoop.hbase.ipc">ServerNotRunningYetException</a>,
                       <a href="../../../../../org/apache/hadoop/hbase/MasterNotRunningException.html" title="class in org.apache.hadoop.hbase">MasterNotRunningException</a>,
@@ -3964,7 +3964,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isActiveMaster</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3080">isActiveMaster</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3082">isActiveMaster</a>()</pre>
 <div class="block">Report whether this master is currently the active master or not.
  If not active master, we are parked on ZK waiting to become active.
 
@@ -3983,7 +3983,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isInitialized</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3094">isInitialized</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3096">isInitialized</a>()</pre>
 <div class="block">Report whether this master has completed with its initialization and is
  ready.  If ready, the master is also the active master.  A standby master
  is never ready.
@@ -4003,7 +4003,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>isInMaintenanceMode</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3104">isInMaintenanceMode</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3106">isInMaintenanceMode</a>()</pre>
 <div class="block">Report whether this master is in maintenance mode.</div>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -4019,7 +4019,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>setInitialized</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3109">setInitialized</a>(boolean&nbsp;isInitialized)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3111">setInitialized</a>(boolean&nbsp;isInitialized)</pre>
 </li>
 </ul>
 <a name="getInitializedEvent--">
@@ -4028,7 +4028,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getInitializedEvent</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureEvent.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureEvent</a>&lt;?&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3114">getInitializedEvent</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureEvent.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureEvent</a>&lt;?&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3116">getInitializedEvent</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getInitializedEvent--">getInitializedEvent</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -4043,7 +4043,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getAverageLoad</h4>
-<pre>public&nbsp;double&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3124">getAverageLoad</a>()</pre>
+<pre>public&nbsp;double&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3126">getAverageLoad</a>()</pre>
 <div class="block">Compute the average load across all region servers.
  Currently, this uses a very naive computation - just uses the number of
  regions being served, ignoring stats about number of requests.</div>
@@ -4059,7 +4059,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getSplitPlanCount</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3139">getSplitPlanCount</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3141">getSplitPlanCount</a>()</pre>
 </li>
 </ul>
 <a name="getMergePlanCount--">
@@ -4068,7 +4068,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMergePlanCount</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3146">getMergePlanCount</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3148">getMergePlanCount</a>()</pre>
 </li>
 </ul>
 <a name="registerService-com.google.protobuf.Service-">
@@ -4077,7 +4077,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>registerService</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3151">registerService</a>(com.google.protobuf.Service&nbsp;instance)</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3153">registerService</a>(com.google.protobuf.Service&nbsp;instance)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/RegionServerServices.html#registerService-com.google.protobuf.Service-">RegionServerServices</a></code></span></div>
 <div class="block">Registers a new protocol buffer <code>Service</code> subclass as a coprocessor endpoint to be
  available for handling</div>
@@ -4101,7 +4101,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>constructMaster</h4>
-<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/HMaster.html" title="class in org.apache.hadoop.hbase.master">HMaster</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3176">constructMaster</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Class.html?is-external=true" title="class or interface in java.lang">Class</a>&lt;? extends <a href="../../../../../org/apache/hadoop/hbase/master/HMaster.html" title="class in org.apache.hadoop.hbase.master">HMaster</a>&gt;&nbsp;masterClass,
+<pre>public static&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/HMaster.html" title="class in org.apache.hadoop.hbase.master">HMaster</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3178">constructMaster</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Class.html?is-external=true" title="class or interface in java.lang">Class</a>&lt;? extends <a href="../../../../../org/apache/hadoop/hbase/master/HMaster.html" title="class in org.apache.hadoop.hbase.master">HMaster</a>&gt;&nbsp;masterClass,
                                       org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 <div class="block">Utility for constructing an instance of the passed HMaster class.</div>
 <dl>
@@ -4118,7 +4118,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>main</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3195">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;args)</pre>
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3197">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;args)</pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../org/apache/hadoop/hbase/master/HMasterCommandLine.html" title="class in org.apache.hadoop.hbase.master"><code>HMasterCommandLine</code></a></dd>
@@ -4131,7 +4131,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getHFileCleaner</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/cleaner/HFileCleaner.html" title="class in org.apache.hadoop.hbase.master.cleaner">HFileCleaner</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3201">getHFileCleaner</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/cleaner/HFileCleaner.html" title="class in org.apache.hadoop.hbase.master.cleaner">HFileCleaner</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3203">getHFileCleaner</a>()</pre>
 </li>
 </ul>
 <a name="getLogCleaner--">
@@ -4140,7 +4140,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogCleaner</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/cleaner/LogCleaner.html" title="class in org.apache.hadoop.hbase.master.cleaner">LogCleaner</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3205">getLogCleaner</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/cleaner/LogCleaner.html" title="class in org.apache.hadoop.hbase.master.cleaner">LogCleaner</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3207">getLogCleaner</a>()</pre>
 </li>
 </ul>
 <a name="getSnapshotManager--">
@@ -4149,7 +4149,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getSnapshotManager</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/snapshot/SnapshotManager.html" title="class in org.apache.hadoop.hbase.master.snapshot">SnapshotManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3213">getSnapshotManager</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/snapshot/SnapshotManager.html" title="class in org.apache.hadoop.hbase.master.snapshot">SnapshotManager</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3215">getSnapshotManager</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getSnapshotManager--">getSnapshotManager</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -4164,7 +4164,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getMasterProcedureManagerHost</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure/MasterProcedureManagerHost.html" title="class in org.apache.hadoop.hbase.procedure">MasterProcedureManagerHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3221">getMasterProcedureManagerHost</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure/MasterProcedureManagerHost.html" title="class in org.apache.hadoop.hbase.procedure">MasterProcedureManagerHost</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3223">getMasterProcedureManagerHost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getMasterProcedureManagerHost--">getMasterProcedureManagerHost</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -4179,7 +4179,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getClusterSchema</h4>
-<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ClusterSchema.html" title="interface in org.apache.hadoop.hbase.master">ClusterSchema</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3226">getClusterSchema</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../org/apache/hadoop/hbase/master/ClusterSchema.html" title="interface in org.apache.hadoop.hbase.master">ClusterSchema</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3228">getClusterSchema</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getClusterSchema--">getClusterSchema</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a></code></dd>
@@ -4194,7 +4194,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>createNamespace</h4>
-<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3238">createNamespace</a>(<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;namespaceDescriptor,
+<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3240">createNamespace</a>(<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;namespaceDescriptor,
                      long&nbsp;nonceGroup,
                      long&nbsp;nonce)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -4218,7 +4218,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>modifyNamespace</h4>
-<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3274">modifyNamespace</a>(<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;newNsDescriptor,
+<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3276">modifyNamespace</a>(<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;newNsDescriptor,
                      long&nbsp;nonceGroup,
                      long&nbsp;nonce)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -4241,7 +4241,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteNamespace</h4>
-<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3312">deleteNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name,
+<pre>long&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3314">deleteNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name,
                      long&nbsp;nonceGroup,
                      long&nbsp;nonce)
               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -4264,7 +4264,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getNamespace</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3347">getNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
+<pre><a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3349">getNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Get a Namespace</div>
 <dl>
@@ -4283,7 +4283,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getNamespaces</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3359">getNamespaces</a>()
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/NamespaceDescriptor.html" title="class in org.apache.hadoop.hbase">NamespaceDescriptor</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3361">getNamespaces</a>()
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Get all Namespaces</div>
 <dl>
@@ -4300,7 +4300,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>listTableNamesByNamespace</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3373">listTableNamesByNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3375">listTableNamesByNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
                                           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#listTableNamesByNamespace-java.lang.String-">MasterServices</a></code></span></div>
 <div class="block">Get list of table names by namespace</div>
@@ -4322,7 +4322,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>listTableDescriptorsByNamespace</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3379">listTableDescriptorsByNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/client/TableDescriptor.html" title="interface in org.apache.hadoop.hbase.client">TableDescriptor</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3381">listTableDescriptorsByNamespace</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)
                                                       throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#listTableDescriptorsByNamespace-java.lang.String-">MasterServices</a></code></span></div>
 <div class="block">Get list of table descriptors by namespace</div>
@@ -4344,7 +4344,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>abortProcedure</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3385">abortProcedure</a>(long&nbsp;procId,
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3387">abortProcedure</a>(long&nbsp;procId,
                               boolean&nbsp;mayInterruptIfRunning)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#abortProcedure-long-boolean-">MasterServices</a></code></span></div>
@@ -4368,7 +4368,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getProcedures</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3401">getProcedures</a>()
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3403">getProcedures</a>()
                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getProcedures--">MasterServices</a></code></span></div>
 <div class="block">Get procedures</div>
@@ -4388,7 +4388,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>getLocks</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResource.html" title="class in org.apache.hadoop.hbase.procedure2">LockedResource</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3417">getLocks</a>()
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResource.html" title="class in org.apache.hadoop.hbase.procedure2">LockedResource</a>&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/master/HMaster.html#line.3419">getLocks</a>()
                               throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/master/MasterServices.html#getLocks--">MasterServices</a></code></span></div>
 <div class="block">Get locks</div>
@@ -4408,7 +4408,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/master/MasterServices
 <ul class="blockList">
 <li class="blockList">
 <h4>listTableDescriptors</h4>
-<pre>public&

<TRUNCATED>

[20/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="lin

<TRUNCATED>

[07/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
index 9b964f6..98ef11a 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.LeaseRecovery.html
@@ -105,7 +105,7 @@
 <span class="sourceLineNo">097</span> * will first be initialized to the oldest file's tracker(which is stored in the trailer), using the<a name="line.97"></a>
 <span class="sourceLineNo">098</span> * method {@link ProcedureStoreTracker#resetTo(ProcedureStoreTracker, boolean)}, and then merge it<a name="line.98"></a>
 <span class="sourceLineNo">099</span> * with the tracker of every newer wal files, using the<a name="line.99"></a>
-<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)}.<a name="line.100"></a>
+<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker)}.<a name="line.100"></a>
 <span class="sourceLineNo">101</span> * If we find out<a name="line.101"></a>
 <span class="sourceLineNo">102</span> * that all the modified procedures for the oldest wal file are modified or deleted in newer wal<a name="line.102"></a>
 <span class="sourceLineNo">103</span> * files, then we can delete it. This is because that, every time we call<a name="line.103"></a>
@@ -1181,244 +1181,243 @@
 <span class="sourceLineNo">1173</span>    }<a name="line.1173"></a>
 <span class="sourceLineNo">1174</span><a name="line.1174"></a>
 <span class="sourceLineNo">1175</span>    // compute the holding tracker.<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>    //  - the first WAL is used for the 'updates'<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span>    //  - the global tracker is passed in first to decide which procedures are not<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>    //    exist anymore, so we can mark them as deleted in holdingCleanupTracker.<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>    //    Only global tracker have the whole picture here.<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>    //  - the other WALs are scanned to remove procs already updated in a newer wal.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>    //    If it is updated in a newer wal, we can mark it as delelted in holdingCleanupTracker<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span>    //    But, we can not delete it if it was shown deleted in the newer wal, as said<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>    //    above.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span>    // TODO: exit early if holdingCleanupTracker.isEmpty()<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>    //Passing in the global tracker, we can delete the procedures not in the global<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>    //tracker, because they are deleted in the later logs<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    holdingCleanupTracker.setDeletedIfModifiedInBoth(storeTracker, true);<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    for (int i = 1, size = logs.size() - 1; i &lt; size; ++i) {<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>      // Set deleteIfNotExists to false since a single log's tracker is passed in.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>      // Since a specific procedure may not show up in the log at all(not executed or<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span>      // updated during the time), we can not delete the procedure just because this log<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>      // don't have the info of the procedure. We can delete the procedure only if<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>      // in this log's tracker, it was cleanly showed that the procedure is modified or deleted<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>      // in the corresponding BitSetNode.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(logs.get(i).getTracker(), false);<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span>    }<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>  }<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span><a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>  /**<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>   */<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    if (logs.size() &lt;= 1) {<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>      return;<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    }<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span><a name="line.1207"></a>
-<span class="sourceLineNo">1208</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span><a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>    boolean removed = false;<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    while (logs.size() &gt; 1) {<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>        break;<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>      }<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>      removeLogFile(log, walArchiveDir);<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>      removed = true;<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    }<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span><a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>    if (removed) {<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>      buildHoldingCleanupTracker();<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>    }<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>  }<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span><a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>    try {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span>      LOG.trace("Removing log={}", log);<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>      log.removeFile(walArchiveDir);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>      logs.remove(log);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>    } catch (IOException e) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>      return false;<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    }<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>    return true;<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>  }<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span><a name="line.1238"></a>
-<span class="sourceLineNo">1239</span>  // ==========================================================================<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  //  FileSystem Log Files helpers<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>  // ==========================================================================<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>  public Path getWALDir() {<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>    return this.walDir;<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span>  }<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span><a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>  @VisibleForTesting<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span>  Path getWalArchiveDir() {<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    return this.walArchiveDir;<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>  }<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span><a name="line.1250"></a>
-<span class="sourceLineNo">1251</span>  public FileSystem getFileSystem() {<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    return this.fs;<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>  }<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span><a name="line.1254"></a>
-<span class="sourceLineNo">1255</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  private static long getLogIdFromName(final String name) {<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>    int end = name.lastIndexOf(".log");<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>    return Long.parseLong(name.substring(start, end));<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span>  }<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span><a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    @Override<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>    public boolean accept(Path path) {<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span>      String name = path.getName();<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>    }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>  };<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span><a name="line.1272"></a>
-<span class="sourceLineNo">1273</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>    @Override<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>      return Long.compare(aId, bId);<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>    }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>  };<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span><a name="line.1282"></a>
-<span class="sourceLineNo">1283</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>    try {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>      return files;<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>    } catch (FileNotFoundException e) {<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>      return null;<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>    }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>  }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span><a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>  /**<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>   * the file set by log id.<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   * @return Max-LogID of the specified log file set<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>   */<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>      return 0L;<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>    }<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>  }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span><a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>  /**<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   * the file set by log id.<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   * @return Max-LogID of the specified log file set<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   */<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>      return 0L;<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>    }<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>    long maxLogId = 0;<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      final Path logPath = logFiles[i].getPath();<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>      if (!isRunning()) {<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>        throw new IOException("wal aborting");<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span>      }<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span><a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>      if (log != null) {<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>        this.logs.add(log);<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>      }<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    }<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span>    initTrackerFromOldLogs();<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    return maxLogId;<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>  }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span>  /**<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>   */<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>  private void initTrackerFromOldLogs() {<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span>      return;<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>    ProcedureWALFile log = logs.getLast();<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (!log.getTracker().isPartial()) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      storeTracker.resetTo(log.getTracker());<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>    } else {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>      storeTracker.reset();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      storeTracker.setPartialFlag(true);<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>    }<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>  }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span><a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>  /**<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>   * Loads given log file and it's tracker.<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>   */<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>      throws IOException {<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span>    if (logFile.getLen() == 0) {<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span>      log.removeFile(walArchiveDir);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>      return null;<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>    }<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span>    try {<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>      log.open();<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>      log.removeFile(walArchiveDir);<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      return null;<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span>    } catch (IOException e) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      LOG.error(msg, e);<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span>      throw new IOException(msg, e);<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span>    }<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span><a name="line.1373"></a>
-<span class="sourceLineNo">1374</span>    try {<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>      log.readTracker();<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>    } catch (IOException e) {<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>      log.getTracker().reset();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>      log.getTracker().setPartialFlag(true);<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span>    }<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span><a name="line.1381"></a>
-<span class="sourceLineNo">1382</span>    log.close();<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span>    return log;<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span>  }<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span><a name="line.1385"></a>
-<span class="sourceLineNo">1386</span>  /**<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span>   * For testing parse and profiling.<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span>   */<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span>  public static void main(String [] args) throws IOException {<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span>    if (args == null || args.length != 1) {<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span>      System.exit(-1);<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span>    }<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span>        @Override<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span>          // no-op<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span>        }<a name="line.1403"></a>
-<span class="sourceLineNo">1404</span>      });<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span>    try {<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span>      store.start(16);<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span>      pe.init(1, true);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span>    } finally {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span>      store.stop(true);<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span>    }<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span>  }<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span>}<a name="line.1413"></a>
+<span class="sourceLineNo">1176</span>    // - the first WAL is used for the 'updates'<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    // - the global tracker will be used to determine whether a procedure has been deleted<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    // - other trackers will be used to determine whether a procedure has been updated, as a deleted<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>    // procedure can always be detected by checking the global tracker, we can save the deleted<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span>    // checks when applying other trackers<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span>    holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span>    // the logs is a linked list, so avoid calling get(index) on it.<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span>    Iterator&lt;ProcedureWALFile&gt; iter = logs.iterator();<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span>    // skip the tracker for the first file when creating the iterator.<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>    iter.next();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>    ProcedureStoreTracker tracker = iter.next().getTracker();<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>    // testing iter.hasNext after calling iter.next to skip applying the tracker for last file,<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>    // which is just the storeTracker above.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>    while (iter.hasNext()) {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker);<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>      if (holdingCleanupTracker.isEmpty()) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>        break;<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>      }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span>      iter.next();<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>    }<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span>  }<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span><a name="line.1198"></a>
+<span class="sourceLineNo">1199</span>  /**<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   */<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>    if (logs.size() &lt;= 1) {<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>      return;<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>    }<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span><a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span><a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    boolean removed = false;<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>    while (logs.size() &gt; 1) {<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>        break;<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>      }<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>      removeLogFile(log, walArchiveDir);<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span>      removed = true;<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>    }<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span><a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>    if (removed) {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>      buildHoldingCleanupTracker();<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    }<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>  }<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span><a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>    try {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>      LOG.trace("Removing log={}", log);<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      log.removeFile(walArchiveDir);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>      logs.remove(log);<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>    } catch (IOException e) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>      return false;<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    }<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>    return true;<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span>  }<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span><a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>  // ==========================================================================<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>  //  FileSystem Log Files helpers<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>  // ==========================================================================<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  public Path getWALDir() {<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>    return this.walDir;<a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span><a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>  @VisibleForTesting<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  Path getWalArchiveDir() {<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span>    return this.walArchiveDir;<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span>  }<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span><a name="line.1249"></a>
+<span class="sourceLineNo">1250</span>  public FileSystem getFileSystem() {<a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    return this.fs;<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>  }<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span><a name="line.1253"></a>
+<span class="sourceLineNo">1254</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>  }<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span><a name="line.1257"></a>
+<span class="sourceLineNo">1258</span>  private static long getLogIdFromName(final String name) {<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span>    int end = name.lastIndexOf(".log");<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span>    return Long.parseLong(name.substring(start, end));<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span><a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>    @Override<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span>    public boolean accept(Path path) {<a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>      String name = path.getName();<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    }<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  };<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    @Override<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>      return Long.compare(aId, bId);<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    }<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  };<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>    try {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span>      return files;<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>    } catch (FileNotFoundException e) {<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>      return null;<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>    }<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>  }<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span><a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>  /**<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   * the file set by log id.<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   * @return Max-LogID of the specified log file set<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>   */<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>      return 0L;<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>    }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>  }<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span><a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>  /**<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   * the file set by log id.<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   * @return Max-LogID of the specified log file set<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   */<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>      return 0L;<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>    }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span>    long maxLogId = 0;<a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>      final Path logPath = logFiles[i].getPath();<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>      if (!isRunning()) {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>        throw new IOException("wal aborting");<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      }<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span><a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span>      if (log != null) {<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>        this.logs.add(log);<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>      }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    }<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    initTrackerFromOldLogs();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    return maxLogId;<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>  }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span><a name="line.1331"></a>
+<span class="sourceLineNo">1332</span>  /**<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>   */<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span>  private void initTrackerFromOldLogs() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>      return;<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>    }<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    ProcedureWALFile log = logs.getLast();<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>    if (!log.getTracker().isPartial()) {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span>      storeTracker.resetTo(log.getTracker());<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>    } else {<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>      storeTracker.reset();<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>      storeTracker.setPartialFlag(true);<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>    }<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>  }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>  /**<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>   * Loads given log file and it's tracker.<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>   */<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>      throws IOException {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>    if (logFile.getLen() == 0) {<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>      log.removeFile(walArchiveDir);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>      return null;<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span>    }<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span>    try {<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>      log.open();<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span>      log.removeFile(walArchiveDir);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>      return null;<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>    } catch (IOException e) {<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span>      LOG.error(msg, e);<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      throw new IOException(msg, e);<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span><a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>    try {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>      log.readTracker();<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span>    } catch (IOException e) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>      log.getTracker().reset();<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span>      log.getTracker().setPartialFlag(true);<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span>    }<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span><a name="line.1380"></a>
+<span class="sourceLineNo">1381</span>    log.close();<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span>    return log;<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span>  }<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span><a name="line.1384"></a>
+<span class="sourceLineNo">1385</span>  /**<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span>   * For testing parse and profiling.<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span>   */<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span>  public static void main(String [] args) throws IOException {<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span>    if (args == null || args.length != 1) {<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span>      System.exit(-1);<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span>    }<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span>        @Override<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span>          // no-op<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span>        }<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span>      });<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span>    try {<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span>      store.start(16);<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span>      pe.init(1, true);<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span>    } finally {<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span>      store.stop(true);<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span>    }<a name="line.1410"></a>
+<span class="sourceLineNo">1411</span>  }<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span>}<a name="line.1412"></a>
 
 
 


[39/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
index b7a2a38..6591d6c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.RedirectServlet.html
@@ -1789,2174 +1789,2176 @@
 <span class="sourceLineNo">1781</span>      LOG.debug("Master has not been initialized, don't run region normalizer.");<a name="line.1781"></a>
 <span class="sourceLineNo">1782</span>      return false;<a name="line.1782"></a>
 <span class="sourceLineNo">1783</span>    }<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span><a name="line.1784"></a>
-<span class="sourceLineNo">1785</span>    if (isInMaintenanceMode()) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span>      return false;<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span>    }<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span><a name="line.1789"></a>
-<span class="sourceLineNo">1790</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1791"></a>
-<span class="sourceLineNo">1792</span>      return false;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span>    }<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span><a name="line.1794"></a>
-<span class="sourceLineNo">1795</span>    synchronized (this.normalizer) {<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span>      // Don't run the normalizer concurrently<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1797"></a>
-<span class="sourceLineNo">1798</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1798"></a>
-<span class="sourceLineNo">1799</span><a name="line.1799"></a>
-<span class="sourceLineNo">1800</span>      Collections.shuffle(allEnabledTables);<a name="line.1800"></a>
+<span class="sourceLineNo">1784</span>    if (this.getServerManager().isClusterShutdown()) {<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span>      LOG.info("Cluster is shutting down, don't run region normalizer.");<a name="line.1785"></a>
+<span class="sourceLineNo">1786</span>      return false;<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span>    }<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span>    if (isInMaintenanceMode()) {<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span>      return false;<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span>    }<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span>      return false;<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span>    }<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span><a name="line.1796"></a>
+<span class="sourceLineNo">1797</span>    synchronized (this.normalizer) {<a name="line.1797"></a>
+<span class="sourceLineNo">1798</span>      // Don't run the normalizer concurrently<a name="line.1798"></a>
+<span class="sourceLineNo">1799</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1800"></a>
 <span class="sourceLineNo">1801</span><a name="line.1801"></a>
-<span class="sourceLineNo">1802</span>      for (TableName table : allEnabledTables) {<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span>        if (isInMaintenanceMode()) {<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span>          return false;<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span>        }<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span><a name="line.1807"></a>
-<span class="sourceLineNo">1808</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span>          continue;<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span>        }<a name="line.1814"></a>
-<span class="sourceLineNo">1815</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span>        if (plans != null) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span>          for (NormalizationPlan plan : plans) {<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span>              splitPlanCount++;<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span>              mergePlanCount++;<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span>            }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span>          }<a name="line.1824"></a>
-<span class="sourceLineNo">1825</span>        }<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span>      }<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span>    }<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span>    // Return true indicating a success.<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span>    return true;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span>  }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span>  /**<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span>   */<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span>  @Override<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span>  public String getClientIdAuditPrefix() {<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span>  }<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span><a name="line.1841"></a>
-<span class="sourceLineNo">1842</span>  /**<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span>   * Switch for the background CatalogJanitor thread.<a name="line.1843"></a>
-<span class="sourceLineNo">1844</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span>   * if disabled.<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span>   */<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span>  }<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span><a name="line.1851"></a>
-<span class="sourceLineNo">1852</span>  @Override<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span>  public long mergeRegions(<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span>      final RegionInfo[] regionsToMerge,<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span>      final boolean forcible,<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span>      final long nonceGroup,<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span>      final long nonce) throws IOException {<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span>    checkInitialized();<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span><a name="line.1859"></a>
-<span class="sourceLineNo">1860</span>    assert(regionsToMerge.length == 2);<a name="line.1860"></a>
+<span class="sourceLineNo">1802</span>      Collections.shuffle(allEnabledTables);<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span><a name="line.1803"></a>
+<span class="sourceLineNo">1804</span>      for (TableName table : allEnabledTables) {<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span>        if (isInMaintenanceMode()) {<a name="line.1805"></a>
+<span class="sourceLineNo">1806</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span>          return false;<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span>        }<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span><a name="line.1809"></a>
+<span class="sourceLineNo">1810</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span>          continue;<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span>        }<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span>        if (plans != null) {<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span>          for (NormalizationPlan plan : plans) {<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1821"></a>
+<span class="sourceLineNo">1822</span>              splitPlanCount++;<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span>              mergePlanCount++;<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span>            }<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span>          }<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span>      }<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span>    }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span>    // Return true indicating a success.<a name="line.1831"></a>
+<span class="sourceLineNo">1832</span>    return true;<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span>  }<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span><a name="line.1834"></a>
+<span class="sourceLineNo">1835</span>  /**<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span>   */<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span>  @Override<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span>  public String getClientIdAuditPrefix() {<a name="line.1839"></a>
+<span class="sourceLineNo">1840</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span>  }<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span><a name="line.1843"></a>
+<span class="sourceLineNo">1844</span>  /**<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span>   * Switch for the background CatalogJanitor thread.<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span>   * if disabled.<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span>   */<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1850"></a>
+<span class="sourceLineNo">1851</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span>  }<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span><a name="line.1853"></a>
+<span class="sourceLineNo">1854</span>  @Override<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span>  public long mergeRegions(<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span>      final RegionInfo[] regionsToMerge,<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span>      final boolean forcible,<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span>      final long nonceGroup,<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span>      final long nonce) throws IOException {<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span>    checkInitialized();<a name="line.1860"></a>
 <span class="sourceLineNo">1861</span><a name="line.1861"></a>
-<span class="sourceLineNo">1862</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span>    }<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span><a name="line.1866"></a>
-<span class="sourceLineNo">1867</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span>      throw new IOException (<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1869"></a>
-<span class="sourceLineNo">1870</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span>    }<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span><a name="line.1872"></a>
-<span class="sourceLineNo">1873</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span>      throw new MergeRegionException(<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span>    }<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span><a name="line.1877"></a>
-<span class="sourceLineNo">1878</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1878"></a>
-<span class="sourceLineNo">1879</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span>      @Override<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span>      protected void run() throws IOException {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span><a name="line.1883"></a>
-<span class="sourceLineNo">1884</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span><a name="line.1886"></a>
-<span class="sourceLineNo">1887</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span>          regionsToMerge, forcible));<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span><a name="line.1889"></a>
-<span class="sourceLineNo">1890</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span>      }<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span><a name="line.1892"></a>
-<span class="sourceLineNo">1893</span>      @Override<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span>      protected String getDescription() {<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span>        return "MergeTableProcedure";<a name="line.1895"></a>
-<span class="sourceLineNo">1896</span>      }<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span>    });<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span>  }<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span>  @Override<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1901"></a>
-<span class="sourceLineNo">1902</span>      final long nonceGroup, final long nonce)<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span>  throws IOException {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span>    checkInitialized();<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1905"></a>
-<span class="sourceLineNo">1906</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>      @Override<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>      protected void run() throws IOException {<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span><a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>        // Execute the operation asynchronously<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span><a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>      @Override<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span>      protected String getDescription() {<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>        return "SplitTableProcedure";<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>      }<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>    });<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>  // Replace with an async implementation from which you can get<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span>  // a success/failure result.<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>  @VisibleForTesting<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span><a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    RegionInfo hri;<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>    if (regionState != null) {<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>      hri = regionState.getRegion();<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>    } else {<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span><a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>    ServerName dest;<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>        : new ArrayList&lt;&gt;(1);<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>      LOG.info(<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>              + " because the server is in exclude list");<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>      destServerName = null;<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>    }<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>        "choosing a server at random");<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>      exclude.add(regionState.getServerName());<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>      if (dest == null) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>        return;<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span>      }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    } else {<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span>      if (dest == null) {<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>        return;<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>      }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span>      // TODO: What is this? I don't get it.<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>        // regions on master.<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>          + " because it should not be on master");<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>        return;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span>      }<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    }<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span><a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>      return;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>    }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>    // Now we can do the move<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span><a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>    try {<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      checkInitialized();<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>      if (this.cpHost != null) {<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>      }<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span>      // closed<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span><a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>      try {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        // Is this going to work? Will we throw exception on error?<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        future.get();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>        throw new HBaseIOException(e);<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>      }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span>      if (this.cpHost != null) {<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>    } catch (IOException ioe) {<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      if (ioe instanceof HBaseIOException) {<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        throw (HBaseIOException)ioe;<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      }<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>      throw new HBaseIOException(ioe);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span>  }<a name="line.2014"></a>
-<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>  @Override<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>  public long createTable(<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>      final TableDescriptor tableDescriptor,<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>      final byte [][] splitKeys,<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>      final long nonceGroup,<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span>      final long nonce) throws IOException {<a name="line.2021"></a>
-<span class="sourceLineNo">2022</span>    checkInitialized();<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span><a name="line.2023"></a>
-<span class="sourceLineNo">2024</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span><a name="line.2026"></a>
-<span class="sourceLineNo">2027</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2027"></a>
-<span class="sourceLineNo">2028</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2028"></a>
-<span class="sourceLineNo">2029</span><a name="line.2029"></a>
-<span class="sourceLineNo">2030</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span>      @Override<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span>      protected void run() throws IOException {<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2034"></a>
-<span class="sourceLineNo">2035</span><a name="line.2035"></a>
-<span class="sourceLineNo">2036</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2036"></a>
+<span class="sourceLineNo">1862</span>    assert(regionsToMerge.length == 2);<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span><a name="line.1863"></a>
+<span class="sourceLineNo">1864</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span><a name="line.1868"></a>
+<span class="sourceLineNo">1869</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span>      throw new IOException (<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span>    }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span><a name="line.1874"></a>
+<span class="sourceLineNo">1875</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span>      throw new MergeRegionException(<a name="line.1876"></a>
+<span class="sourceLineNo">1877</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span>    }<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span><a name="line.1879"></a>
+<span class="sourceLineNo">1880</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span>      @Override<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span>      protected void run() throws IOException {<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span><a name="line.1888"></a>
+<span class="sourceLineNo">1889</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span>          regionsToMerge, forcible));<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span><a name="line.1891"></a>
+<span class="sourceLineNo">1892</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span>      }<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span><a name="line.1894"></a>
+<span class="sourceLineNo">1895</span>      @Override<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span>      protected String getDescription() {<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span>        return "MergeTableProcedure";<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span>      }<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span>    });<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span>  }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span><a name="line.1901"></a>
+<span class="sourceLineNo">1902</span>  @Override<a name="line.1902"></a>
+<span class="sourceLineNo">1903</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span>      final long nonceGroup, final long nonce)<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span>  throws IOException {<a name="line.1905"></a>
+<span class="sourceLineNo">1906</span>    checkInitialized();<a name="line.1906"></a>
+<span class="sourceLineNo">1907</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>      @Override<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>      protected void run() throws IOException {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        // Execute the operation asynchronously<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>      }<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>      @Override<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>      protected String getDescription() {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span>        return "SplitTableProcedure";<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>      }<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>    });<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  }<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span><a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span>  // Replace with an async implementation from which you can get<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>  // a success/failure result.<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span>  @VisibleForTesting<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    RegionInfo hri;<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    if (regionState != null) {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      hri = regionState.getRegion();<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    } else {<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span>    }<a name="line.1938"></a>
+<span class="sourceLineNo">1939</span><a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>    ServerName dest;<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>        : new ArrayList&lt;&gt;(1);<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>      LOG.info(<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span>              + " because the server is in exclude list");<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>      destServerName = null;<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    }<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span>        "choosing a server at random");<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>      exclude.add(regionState.getServerName());<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span>      if (dest == null) {<a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        return;<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>      }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span>    } else {<a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>      if (dest == null) {<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>        return;<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      }<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>      // TODO: What is this? I don't get it.<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>        // regions on master.<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>          + " because it should not be on master");<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>        return;<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>      }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span>    }<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span><a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>      return;<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>    }<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span><a name="line.1983"></a>
+<span class="sourceLineNo">1984</span>    // Now we can do the move<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span><a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>    try {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      checkInitialized();<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      if (this.cpHost != null) {<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span>      }<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      // closed<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span><a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      try {<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>        // Is this going to work? Will we throw exception on error?<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        future.get();<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        throw new HBaseIOException(e);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>      if (this.cpHost != null) {<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>      }<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>    } catch (IOException ioe) {<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>      if (ioe instanceof HBaseIOException) {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>        throw (HBaseIOException)ioe;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>      }<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      throw new HBaseIOException(ioe);<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>    }<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>  }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>  @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>  public long createTable(<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      final TableDescriptor tableDescriptor,<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>      final byte [][] splitKeys,<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>      final long nonceGroup,<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>      final long nonce) throws IOException {<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span>    checkInitialized();<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span><a name="line.2025"></a>
+<span class="sourceLineNo">2026</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2027"></a>
+<span class="sourceLineNo">2028</span><a name="line.2028"></a>
+<span class="sourceLineNo">2029</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span><a name="line.2031"></a>
+<span class="sourceLineNo">2032</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span>      @Override<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span>      protected void run() throws IOException {<a name="line.2035"></a>
+<span class="sourceLineNo">2036</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2036"></a>
 <span class="sourceLineNo">2037</span><a name="line.2037"></a>
-<span class="sourceLineNo">2038</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span>        //<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2042"></a>
-<span class="sourceLineNo">2043</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span>        submitProcedure(new CreateTableProcedure(<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span>        latch.await();<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span>      }<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span><a name="line.2050"></a>
-<span class="sourceLineNo">2051</span>      @Override<a name="line.2051"></a>
-<span class="sourceLineNo">2052</span>      protected String getDescription() {<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span>        return "CreateTableProcedure";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span>      }<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span>    });<a name="line.2055"></a>
-<span class="sourceLineNo">2056</span>  }<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span><a name="line.2057"></a>
-<span class="sourceLineNo">2058</span>  @Override<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2059"></a>
-<span class="sourceLineNo">2060</span>    if (isStopped()) {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span>      throw new MasterNotRunningException();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span>    }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span>    if (!(tableName.isSystemTable())) {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span>      throw new IllegalArgumentException(<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span>        "Only system table creation can use this createSystemTable API");<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span>    }<a name="line.2068"></a>
-<span class="sourceLineNo">2069</span><a name="line.2069"></a>
-<span class="sourceLineNo">2070</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2070"></a>
+<span class="sourceLineNo">2038</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span><a name="line.2039"></a>
+<span class="sourceLineNo">2040</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2041"></a>
+<span class="sourceLineNo">2042</span>        //<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span>        submitProcedure(new CreateTableProcedure(<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span>        latch.await();<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span>      }<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span><a name="line.2052"></a>
+<span class="sourceLineNo">2053</span>      @Override<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span>      protected String getDescription() {<a name="line.2054"></a>
+<span class="sourceLineNo">2055</span>        return "CreateTableProcedure";<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span>      }<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span>    });<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span>  }<a name="line.2058"></a>
+<span class="sourceLineNo">2059</span><a name="line.2059"></a>
+<span class="sourceLineNo">2060</span>  @Override<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span>    if (isStopped()) {<a name="line.2062"></a>
+<span class="sourceLineNo">2063</span>      throw new MasterNotRunningException();<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span>    }<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span><a name="line.2065"></a>
+<span class="sourceLineNo">2066</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2066"></a>
+<span class="sourceLineNo">2067</span>    if (!(tableName.isSystemTable())) {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span>      throw new IllegalArgumentException(<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span>        "Only system table creation can use this createSystemTable API");<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span>    }<a name="line.2070"></a>
 <span class="sourceLineNo">2071</span><a name="line.2071"></a>
-<span class="sourceLineNo">2072</span>    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2072"></a>
+<span class="sourceLineNo">2072</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2072"></a>
 <span class="sourceLineNo">2073</span><a name="line.2073"></a>
-<span class="sourceLineNo">2074</span>    // This special create table is called locally to master.  Therefore, no RPC means no need<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span>    // to use nonce to detect duplicated RPC call.<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span>    long procId = this.procedureExecutor.submitProcedure(<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span>      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span><a name="line.2078"></a>
-<span class="sourceLineNo">2079</span>    return procId;<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span>  }<a name="line.2080"></a>
-<span class="sourceLineNo">2081</span><a name="line.2081"></a>
-<span class="sourceLineNo">2082</span>  /**<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span>   * Checks whether the table conforms to some sane limits, and configured<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span>   * values (compression, etc) work. Throws an exception if something is wrong.<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span>   * @throws IOException<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span>   */<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span>  private void sanityCheckTableDescriptor(final TableDescriptor htd) throws IOException {<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span>    final String CONF_KEY = "hbase.table.sanity.checks";<a name="line.2088"></a>
-<span class="sourceLineNo">2089</span>    boolean logWarn = false;<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span>    if (!conf.getBoolean(CONF_KEY, true)) {<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span>      logWarn = true;<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span>    }<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span>    String tableVal = htd.getValue(CONF_KEY);<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span>    if (tableVal != null &amp;&amp; !Boolean.valueOf(tableVal)) {<a name="line.2094"></a>
-<span class="sourceLineNo">2095</span>      logWarn = true;<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span>    }<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span><a name="line.2097"></a>
-<span class="sourceLineNo">2098</span>    // check max file size<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span>    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span>    long maxFileSize = htd.getMaxFileSize();<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span>    if (maxFileSize &lt; 0) {<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span>      maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span>    }<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span>    if (maxFileSize &lt; conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span>      String message = "MAX_FILESIZE for table descriptor or "<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span>          + "\"hbase.hregion.max.filesize\" (" + maxFileSize<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span>          + ") is too small, which might cause over splitting into unmanageable "<a name="line.2107"></a>
-<span class="sourceLineNo">2108</span>          + "number of regions.";<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span>    }<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span><a name="line.2111"></a>
-<span class="sourceLineNo">2112</span>    // check flush size<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span>    long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span>    long flushSize = htd.getMemStoreFlushSize();<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span>    if (flushSize &lt; 0) {<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span>      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);<a name="line.2116"></a>
-<span class="sourceLineNo">2117</span>    }<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span>    if (flushSize &lt; conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span>      String message = "MEMSTORE_FLUSHSIZE for table descriptor or "<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span>          + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span>          + " very frequent flushing.";<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2122"></a>
-<span class="sourceLineNo">2123</span>    }<a name="line.2123"></a>
-<span class="sourceLineNo">2124</span><a name="line.2124"></a>
-<span class="sourceLineNo">2125</span>    // check that coprocessors and other specified plugin classes can be loaded<a name="line.2125"></a>
-<span class="sourceLineNo">2126</span>    try {<a name="line.2126"></a>
-<span class="sourceLineNo">2127</span>      checkClassLoading(conf, htd);<a name="line.2127"></a>
-<span class="sourceLineNo">2128</span>    } catch (Exception ex) {<a name="line.2128"></a>
-<span class="sourceLineNo">2129</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);<a name="line.2129"></a>
-<span class="sourceLineNo">2130</span>    }<a name="line.2130"></a>
-<span class="sourceLineNo">2131</span><a name="line.2131"></a>
-<span class="sourceLineNo">2132</span>    // check compression can be loaded<a name="line.2132"></a>
-<span class="sourceLineNo">2133</span>    try {<a name="line.2133"></a>
-<span class="sourceLineNo">2134</span>      checkCompression(htd);<a name="line.2134"></a>
-<span class="sourceLineNo">2135</span>    } catch (IOException e) {<a name="line.2135"></a>
-<span class="sourceLineNo">2136</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2136"></a>
-<span class="sourceLineNo">2137</span>    }<a name="line.2137"></a>
-<span class="sourceLineNo">2138</span><a name="line.2138"></a>
-<span class="sourceLineNo">2139</span>    // check encryption can be loaded<a name="line.2139"></a>
-<span class="sourceLineNo">2140</span>    try {<a name="line.2140"></a>
-<span class="sourceLineNo">2141</span>      checkEncryption(conf, htd);<a name="line.2141"></a>
-<span class="sourceLineNo">2142</span>    } catch (IOException e) {<a name="line.2142"></a>
-<span class="sourceLineNo">2143</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2143"></a>
-<span class="sourceLineNo">2144</span>    }<a name="line.2144"></a>
-<span class="sourceLineNo">2145</span>    // Verify compaction policy<a name="line.2145"></a>
-<span class="sourceLineNo">2146</span>    try{<a name="line.2146"></a>
-<span class="sourceLineNo">2147</span>      checkCompactionPolicy(conf, htd);<a name="line.2147"></a>
-<span class="sourceLineNo">2148</span>    } catch(IOException e){<a name="line.2148"></a>
-<span class="sourceLineNo">2149</span>      warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);<a name="line.2149"></a>
-<span class="sourceLineNo">2150</span>    }<a name="line.2150"></a>
-<span class="sourceLineNo">2151</span>    // check that we have at least 1 CF<a name="line.2151"></a>
-<span class="sourceLineNo">2152</span>    if (htd.getColumnFamilyCount() == 0) {<a name="line.2152"></a>
-<span class="sourceLineNo">2153</span>      String message = "Table should have at least one column family.";<a name="line.2153"></a>
-<span class="sourceLineNo">2154</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2154"></a>
-<span class="sourceLineNo">2155</span>    }<a name="line.2155"></a>
-<span class="sourceLineNo">2156</span><a name="line.2156"></a>
-<span class="sourceLineNo">2157</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2157"></a>
-<span class="sourceLineNo">2158</span>      if (hcd.getTimeToLive() &lt;= 0) {<a name="line.2158"></a>
-<span class="sourceLineNo">2159</span>        String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";<a name="line.2159"></a>
-<span class="sourceLineNo">2160</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2160"></a>
-<span class="sourceLineNo">2161</span>      }<a name="line.2161"></a>
-<span class="sourceLineNo">2162</span><a name="line.2162"></a>
-<span class="sourceLineNo">2163</span>      // check blockSize<a name="line.2163"></a>
-<span class="sourceLineNo">2164</span>      if (hcd.getBlocksize() &lt; 1024 || hcd.getBlocksize() &gt; 16 * 1024 * 1024) {<a name="line.2164"></a>
-<span class="sourceLineNo">2165</span>        String message = "Block size for column family " + hcd.getNameAsString()<a name="line.2165"></a>
-<span class="sourceLineNo">2166</span>            + "  must be between 1K and 16MB.";<a name="line.2166"></a>
-<span class="sourceLineNo">2167</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2167"></a>
-<span class="sourceLineNo">2168</span>      }<a name="line.2168"></a>
-<span class="sourceLineNo">2169</span><a name="line.2169"></a>
-<span class="sourceLineNo">2170</span>      // check versions<a name="line.2170"></a>
-<span class="sourceLineNo">2171</span>      if (hcd.getMinVersions() &lt; 0) {<a name="line.2171"></a>
-<span class="sourceLineNo">2172</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2172"></a>
-<span class="sourceLineNo">2173</span>          + "  must be positive.";<a name="line.2173"></a>
-<span class="sourceLineNo">2174</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2174"></a>
-<span class="sourceLineNo">2175</span>      }<a name="line.2175"></a>
-<span class="sourceLineNo">2176</span>      // max versions already being checked<a name="line.2176"></a>
-<span class="sourceLineNo">2177</span><a name="line.2177"></a>
-<span class="sourceLineNo">2178</span>      // HBASE-13776 Setting illegal versions for ColumnFamilyDescriptor<a name="line.2178"></a>
-<span class="sourceLineNo">2179</span>      //  does not throw IllegalArgumentException<a name="line.2179"></a>
-<span class="sourceLineNo">2180</span>      // check minVersions &lt;= maxVerions<a name="line.2180"></a>
-<span class="sourceLineNo">2181</span>      if (hcd.getMinVersions() &gt; hcd.getMaxVersions()) {<a name="line.2181"></a>
-<span class="sourceLineNo">2182</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2182"></a>
-<span class="sourceLineNo">2183</span>            + " must be less than the Max versions.";<a name="line.2183"></a>
-<span class="sourceLineNo">2184</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2184"></a>
-<span class="sourceLineNo">2185</span>      }<a name="line.2185"></a>
-<span class="sourceLineNo">2186</span><a name="line.2186"></a>
-<span class="sourceLineNo">2187</span>      // check replication scope<a name="line.2187"></a>
-<span class="sourceLineNo">2188</span>      checkReplicationScope(hcd);<a name="line.2188"></a>
-<span class="sourceLineNo">2189</span>      // check bloom filter type<a name="line.2189"></a>
-<span class="sourceLineNo">2190</span>      checkBloomFilterType(hcd);<a name="line.2190"></a>
-<span class="sourceLineNo">2191</span><a name="line.2191"></a>
-<span class="sourceLineNo">2192</span>      // check data replication factor, it can be 0(default value) when user has not explicitly<a name="line.2192"></a>
-<span class="sourceLineNo">2193</span>      // set the value, in this case we use default replication factor set in the file system.<a name="line.2193"></a>
-<span class="sourceLineNo">2194</span>      if (hcd.getDFSReplication() &lt; 0) {<a name="line.2194"></a>
-<span class="sourceLineNo">2195</span>        String message = "HFile Replication for column family " + hcd.getNameAsString()<a name="line.2195"></a>
-<span class="sourceLineNo">2196</span>            + "  must be greater than zero.";<a name="line.2196"></a>
-<span class="sourceLineNo">2197</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2197"></a>
-<span class="sourceLineNo">2198</span>      }<a name="line.2198"></a>
-<span class="sourceLineNo">2199</span><a name="line.2199"></a>
-<span class="sourceLineNo">2200</span>      // TODO: should we check coprocessors and encryption ?<a name="line.2200"></a>
-<span class="sourceLineNo">2201</span>    }<a name="line.2201"></a>
-<span class="sourceLineNo">2202</span>  }<a name="line.2202"></a>
-<span class="sourceLineNo">2203</span><a name="line.2203"></a>
-<span class="sourceLineNo">2204</span>  private void checkReplicationScope(ColumnFamilyDescriptor hcd) throws IOException{<a name="line.2204"></a>
-<span class="sourceLineNo">2205</span>    // check replication scope<a name="line.2205"></a>
-<span class="sourceLineNo">2206</span>    WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(hcd.getScope());<a name="line.2206"></a>
-<span class="sourceLineNo">2207</span>    if (scop == null) {<a name="line.2207"></a>
-<span class="sourceLineNo">2208</span>      String message = "Replication scope for column family "<a name="line.2208"></a>
-<span class="sourceLineNo">2209</span>          + hcd.getNameAsString() + " is " + hcd.getScope() + " which is invalid.";<a name="line.2209"></a>
-<span class="sourceLineNo">2210</span><a name="line.2210"></a>
-<span class="sourceLineNo">2211</span>      LOG.error(message);<a name="line.2211"></a>
-<span class="sourceLineNo">2212</span>      throw new DoNotRetryIOException(message);<a name="line.2212"></a>
-<span class="sourceLineNo">2213</span>    }<a name="line.2213"></a>
-<span class="sourceLineNo">2214</span>  }<a name="line.2214"></a>
-<span class="sourceLineNo">2215</span><a name="line.2215"></a>
-<span class="sourceLineNo">2216</span>  private void checkCompactionPolicy(Configuration conf, TableDescriptor htd)<a name="line.2216"></a>
-<span class="sourceLineNo">2217</span>      throws IOException {<a name="line.2217"></a>
-<span class="sourceLineNo">2218</span>    // FIFO compaction has some requirements<a name="line.2218"></a>
-<span class="sourceLineNo">2219</span>    // Actually FCP ignores periodic major compactions<a name="line.2219"></a>
-<span class="sourceLineNo">2220</span>    String className = htd.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2220"></a>
-<span class="sourceLineNo">2221</span>    if (className == null) {<a name="line.2221"></a>
-<span class="sourceLineNo">2222</span>      className =<a name="line.2222"></a>
-<span class="sourceLineNo">2223</span>          conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,<a name="line.2223"></a>
-<span class="sourceLineNo">2224</span>            ExploringCompactionPolicy.class.getName());<a name="line.2224"></a>
-<span class="sourceLineNo">2225</span>    }<a name="line.2225"></a>
-<span class="sourceLineNo">2226</span><a name="line.2226"></a>
-<span class="sourceLineNo">2227</span>    int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;<a name="line.2227"></a>
-<span class="sourceLineNo">2228</span>    String sv = htd.getValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2228"></a>
-<span class="sourceLineNo">2229</span>    if (sv != null) {<a name="line.2229"></a>
-<span class="sourceLineNo">2230</span>      blockingFileCount = Integer.parseInt(sv);<a name="line.2230"></a>
-<span class="sourceLineNo">2231</span>    } else {<a name="line.2231"></a>
-<span class="sourceLineNo">2232</span>      blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);<a name="line.2232"></a>
-<span class="sourceLineNo">2233</span>    }<a name="line.2233"></a>
-<span class="sourceLineNo">2234</span><a name="line.2234"></a>
-<span class="sourceLineNo">2235</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2235"></a>
-<span class="sourceLineNo">2236</span>      String compactionPolicy =<a name="line.2236"></a>
-<span class="sourceLineNo">2237</span>          hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2237"></a>
-<span class="sourceLineNo">2238</span>      if (compactionPolicy == null) {<a name="line.2238"></a>
-<span class="sourceLineNo">2239</span>        compactionPolicy = className;<a name="line.2239"></a>
-<span class="sourceLineNo">2240</span>      }<a name="line.2240"></a>
-<span class="sourceLineNo">2241</span>      if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {<a name="line.2241"></a>
-<span class="sourceLineNo">2242</span>        continue;<a name="line.2242"></a>
-<span class="sourceLineNo">2243</span>      }<a name="line.2243"></a>
-<span class="sourceLineNo">2244</span>      // FIFOCompaction<a name="line.2244"></a>
-<span class="sourceLineNo">2245</span>      String message = null;<a name="line.2245"></a>
-<span class="sourceLineNo">2246</span><a name="line.2246"></a>
-<span class="sourceLineNo">2247</span>      // 1. Check TTL<a name="line.2247"></a>
-<span class="sourceLineNo">2248</span>      if (hcd.getTimeToLive() == ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {<a name="line.2248"></a>
-<span class="sourceLineNo">2249</span>        message = "Default TTL is not supported for FIFO compaction";<a name="line.2249"></a>
-<span class="sourceLineNo">2250</span>        throw new IOException(message);<a name="line.2250"></a>
-<span class="sourceLineNo">2251</span>      }<a name="line.2251"></a>
-<span class="sourceLineNo">2252</span><a name="line.2252"></a>
-<span class="sourceLineNo">2253</span>      // 2. Check min versions<a name="line.2253"></a>
-<span class="sourceLineNo">2254</span>      if (hcd.getMinVersions() &gt; 0) {<a name="line.2254"></a>
-<span class="sourceLineNo">2255</span>        message = "MIN_VERSION &gt; 0 is not supported for FIFO compaction";<a name="line.2255"></a>
-<span class="sourceLineNo">2256</span>        throw new IOException(message);<a name="line.2256"></a>
-<span class="sourceLineNo">2257</span>      }<a name="line.2257"></a>
-<span class="sourceLineNo">2258</span><a name="line.2258"></a>
-<span class="sourceLineNo">2259</span>      // 3. blocking file count<a name="line.2259"></a>
-<span class="sourceLineNo">2260</span>      sv = hcd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2260"></a>
-<span class="sourceLineNo">2261</span>      if (sv != null) {<a name="line.2261"></a>
-<span class="sourceLineNo">2262</span>        blockingFileCount = Integer.parseInt(sv);<a name="line.2262"></a>
-<span class="sourceLineNo">2263</span>      }<a name="line.2263"></a>
-<span class="sourceLineNo">2264</span>      if (blockingFileCount &lt; 1000) {<a name="line.2264"></a>
-<span class="sourceLineNo">2265</span>        message =<a name="line.2265"></a>
-<span class="sourceLineNo">2266</span>            "Blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount<a name="line.2266"></a>
-<span class="sourceLineNo">2267</span>                + " is below recommended minimum of 1000 for column family "+ hcd.getNameAsString();<a name="line.2267"></a>
-<span class="sourceLineNo">2268</span>        throw new IOException(message);<a name="line.2268"></a>
-<span class="sourceLineNo">2269</span>      }<a name="line.2269"></a>
-<span class="sourceLineNo">2270</span>    }<a name="line.2270"></a>
-<span class="sourceLineNo">2271</span>  }<a name="line.2271"></a>
-<span class="sourceLineNo">2272</span><a name="line.2272"></a>
-<span class="sourceLineNo">2273</span>  private static void checkBloomFilterType(ColumnFamilyDescriptor cfd)<a name="line.2273"></a>
-<span class="sourceLineNo">2274</span>      throws IOException {<a name="line.2274"></a>
-<span class="sourceLineNo">2275</span>    Configuration conf = new CompoundConfiguration().addStringMap(cfd.getConfiguration());<a name="line.2275"></a>
-<span class="sourceLineNo">2276</span>    try {<a name="line.2276"></a>
-<span class="sourceLineNo">2277</span>      BloomFilterUtil.getBloomFilterParam(cfd.getBloomFilterType(), conf);<a name="line.2277"></a>
-<span class="sourceLineNo">2278</span>    } catch (IllegalArgumentException e) {<a name="line.2278"></a>
-<span class="sourceLineNo">2279</span>      throw new DoNotRetryIOException("Failed to get bloom filter param", e);<a name="line.2279"></a>
-<span class="sourceLineNo">2280</span>    }<a name="line.2280"></a>
-<span class="sourceLineNo">2281</span>  }<a name="line.2281"></a>
-<span class="sourceLineNo">2282</span><a name="line.2282"></a>
-<span class="sourceLineNo">2283</span>  // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.<a name="line.2283"></a>
-<span class="sourceLineNo">2284</span>  private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,<a name="line.2284"></a>
-<span class="sourceLineNo">2285</span>      String message, Exception cause) throws IOException {<a name="line.2285"></a>
-<span class="sourceLineNo">2286</span>    if (!logWarn) {<a name="line.2286"></a>
-<span class="sourceLineNo">2287</span>      throw new DoNotRetryIOException(message + " Set " + confKey +<a name="line.2287"></a>
-<span class="sourceLineNo">2288</span>          " to false at conf or table descriptor if you want to bypass sanity checks", cause);<a name="line.2288"></a>
-<span class="sourceLineNo">2289</span>    }<a name="line.2289"></a>
-<span class="sourceLineNo">2290</span>    LOG.warn(message);<a name="line.2290"></a>
-<span class="sourceLineNo">2291</span>  }<a name="line.2291"></a>
-<span class="sourceLineNo">2292</span><a name="line.2292"></a>
-<span class="sourceLineNo">2293</span>  private void startActiveMasterManager(int infoPort) throws KeeperException {<a name="line.2293"></a>
-<span class="sourceLineNo">2294</span>    String backupZNode = ZNodePaths.joinZNode(<a name="line.2294"></a>
-<span class="sourceLineNo">2295</span>      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());<a name="line.2295"></a>
-<span class="sourceLineNo">2296</span>    /*<a name="line.2296"></a>
-<span class="sourceLineNo">2297</span>    * Add a ZNode for ourselves in the backup master directory since we<a name="line.2297"></a>
-<span class="sourceLineNo">2298</span>    * may not become the active master. If so, we want the actual active<a name="line.2298"></a>
-<span class="sourceLineNo">2299</span>    * master to know we are backup masters, so that it won't assign<a name="line.2299"></a>
-<span class="sourceLineNo">2300</span>    * regions to us if so configured.<a name="line.2300"></a>
-<span class="sourceLineNo">2301</span>    *<a name="line.2301"></a>
-<span class="sourceLineNo">2302</span>    * If we become the active master later, ActiveMasterManager will delete<a name="line.2302"></a>
-<span class="sourceLineNo">2303</span>    * this node explicitly.  If we crash before then, ZooKeeper will delete<a name="line.2303"></a>
-<span class="sourceLineNo">2304</span>    * this node for us since it is ephemeral.<a name="line.2304"></a>
-<span class="sourceLineNo">2305</span>    */<a name="line.2305"></a>
-<span class="sourceLineNo">2306</span>    LOG.info("Adding backup master ZNode " + backupZNode);<a name="line.2306"></a>
-<span class="sourceLineNo">2307</span>    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {<a name="line.2307"></a>
-<span class="sourceLineNo">2308</span>      LOG.warn("Failed create of " + backupZNode + " by " + serverName);<a name="line.2308"></a>
-<span class="sourceLineNo">2309</span>    }<a name="line.2309"></a>
-<span class="sourceLineNo">2310</span>    this.activeMasterManager.setInfoPort(infoPort);<a name="line.2310"></a>
-<span class="sourceLineNo">2311</span>    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);<a name="line.2311"></a>
-<span class="sourceLineNo">2312</span>    // If we're a backup master, stall until a primary to write this address<a name="line.2312"></a>
-<span class="sourceLineNo">2313</span>    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {<a name="line.2313"></a>
-<span class="sourceLineNo">2314</span>      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");<a name="line.2314"></a>
-<span class="sourceLineNo">2315</span>      // This will only be a minute or so while the cluster starts up,<a name="line.2315"></a>
-<span class="sourceLineNo">2316</span>      // so don't worry about setting watches on the parent znode<a name="line.2316"></a>
-<span class="sourceLineNo">2317</span>      while (!activeMasterManager.hasActiveMaster()) {<a name="line.2317"></a>
-<span class="sourceLineNo">2318</span>        LOG.debug("Waiting for master address and cluster state znode to be written.");<a name="line.2318"></a>
-<span class="sourceLineNo">2319</span>        Threads.sleep(timeout);<a name="line.2319"></a>
-<span class="sourceLineNo">2320</span>      }<a name="line.2320"></a>
-<span class="sourceLineNo">2321</span>    }<a name="line.2321"></a>
-<span class="sourceLineNo">2322</span>    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");<a name="line.2322"></a>
-<span class="sourceLineNo">2323</span>    status.setDescription("Master startup");<a name="line.2323"></a>
-<span class="sourceLineNo">2324</span>    try {<a name="line.2324"></a>
-<span class="sourceLineNo">2325</span>      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {<a name="line.2325"></a>
-<span class="sourceLineNo">2326</span>        finishActiveMasterInitialization(status);<a name="line.2326"></a>
-<span class="sourceLineNo">2327</span>      }<a name="line.2327"></a>
-<span class="sourceLineNo">2328</span>    } catch (Throwable t) {<a name="line.2328"></a>
-<span class="sourceLineNo">2329</span>      status.setStatus("Failed to become active: " + t.getMessage());<a name="line.2329"></a>
-<span class="sourceLineNo">2330</span>      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);<a name="line.2330"></a>
-<span class="sourceLineNo">2331</span>      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility<a name="line.2331"></a>
-<span class="sourceLineNo">2332</span>      if (t instanceof NoClassDefFoundError &amp;&amp; t.getMessage().<a name="line.2332"></a>
-<span class="sourceLineNo">2333</span>          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {<a name="line.2333"></a>
-<span class="sourceLineNo">2334</span>        // improved error message for this special case<a name="line.2334"></a>
-<span class="sourceLineNo">2335</span>        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +<a name="line.2335"></a>
-<span class="sourceLineNo">2336</span>          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +<a name="line.2336"></a>
-<span class="sourceLineNo">2337</span>          " or change your hadoop jars to start properly", t);<a name="line.2337"></a>
-<span class="sourceLineNo">2338</span>      } else {<a name="line.2338"></a>
-<span class="sourceLineNo">2339</span>        abort("Unhandled exception. Starting shutdown.", t);<a name="line.2339"></a>
-<span class="sourceLineNo">2340</span>      }<a name="line.2340"></a>
-<span class="sourceLineNo">2341</span>    } finally {<a name="line.2341"></a>
-<span class="sourceLineNo">2342</span>      status.cleanup();<a name="line.2342"></a>
-<span class="sourceLineNo">2343</span>    }<a name="line.2343"></a>
-<span class="sourceLineNo">2344</span>  }<a name="line.2344"></a>
-<span class="sourceLineNo">2345</span><a name="line.2345"></a>
-<span class="sourceLineNo">2346</span>  private void checkCompression(final TableDescriptor htd)<a name="line.2346"></a>
-<span class="sourceLineNo">2347</span>  throws IOException {<a name="line.2347"></a>
-<span class="sourceLineNo">2348</span>    if (!this.masterCheckCompression) return;<a name="line.2348"></a>
-<span class="sourceLineNo">2349</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2349"></a>
-<span class="sourceLineNo">2350</span>      checkCompression(hcd);<a name="line.2350"></a>
-<span class="sourceLineNo">2351</span>    }<a name="line.2351"></a>
-<span class="sourceLineNo">2352</span>  }<a name="line.2352"></a>
-<span class="sourceLineNo">2353</span><a name="line.2353"></a>
-<span class="sourceLineNo">2354</span>  private void checkCompression(final ColumnFamilyDescriptor hcd)<a name="line.2354"></a>
-<span class="sourceLineNo">2355</span>  throws IOException {<a name="line.2355"></a>
-<span class="sourceLineNo">2356</span>    if (!this.masterCheckCompression) return;<a name="line.2356"></a>
-<span class="sourceLineNo">2357</span>    CompressionTest.testCompression(hcd.getCompressionType());<a name="line.2357"></a>
-<span class="sourceLineNo">2358</span>    CompressionTest.testCompression(hcd.getCompactionCompressionType());<a name="line.2358"></a>
-<span class="sourceLineNo

<TRUNCATED>

[19/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toReg

<TRUNCATED>

[36/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">

<TRUNCATED>

[22/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a

<TRUNCATED>

[14/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class=

<TRUNCATED>

[45/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
index 9f55028..b7eabfd 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
@@ -18,7 +18,7 @@
     catch(err) {
     }
 //-->
-var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10,"i38":10,"i39":10};
 var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
 var altColor = "altColor";
 var rowColor = "rowColor";
@@ -111,7 +111,7 @@ var activeTableTab = "activeTableTab";
 <br>
 <pre>@InterfaceAudience.Private
  @InterfaceStability.Evolving
-public class <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.41">ProcedureStoreTracker</a>
+public class <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.42">ProcedureStoreTracker</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Keeps track of live procedures.
 
@@ -355,34 +355,45 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 </tr>
 <tr id="i29" class="rowColor">
 <td class="colFirst"><code>void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfDeletedByThem-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfDeletedByThem</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</code>
+<div class="block">For the global tracker, we will use this method to build the holdingCleanupTracker, as the
+ modified flags will be cleared after rolling so we only need to test the deleted flags.</div>
+</td>
+</tr>
+<tr id="i30" class="altColor">
+<td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-">setDeletedIfModified</a></span>(long...&nbsp;procId)</code>
 <div class="block">Set the given bit for the procId to delete if it was modified before.</div>
 </td>
 </tr>
-<tr id="i30" class="altColor">
+<tr id="i31" class="rowColor">
 <td class="colFirst"><code>void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-">setDeletedIfModifiedInBoth</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-                          boolean&nbsp;globalTracker)</code>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfModifiedInBoth</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</code>
 <div class="block">Similar with <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-"><code>setDeletedIfModified(long...)</code></a>, but here the <code>procId</code> are given by
  the <code>tracker</code>.</div>
 </td>
 </tr>
-<tr id="i31" class="rowColor">
+<tr id="i32" class="altColor">
+<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeleteIf-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-java.util.function.BiFunction-">setDeleteIf</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+           <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/BiFunction.html?is-external=true" title="class or interface in java.util.function">BiFunction</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;func)</code>&nbsp;</td>
+</tr>
+<tr id="i33" class="rowColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setKeepDeletes-boolean-">setKeepDeletes</a></span>(boolean&nbsp;keepDeletes)</code>&nbsp;</td>
 </tr>
-<tr id="i32" class="altColor">
+<tr id="i34" class="altColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setMinMaxModifiedProcIds-long-long-">setMinMaxModifiedProcIds</a></span>(long&nbsp;min,
                         long&nbsp;max)</code>
 <div class="block">Will be called when restarting where we need to rebuild the ProcedureStoreTracker.</div>
 </td>
 </tr>
-<tr id="i33" class="rowColor">
+<tr id="i35" class="rowColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setPartialFlag-boolean-">setPartialFlag</a></span>(boolean&nbsp;isPartial)</code>&nbsp;</td>
 </tr>
-<tr id="i34" class="altColor">
+<tr id="i36" class="altColor">
 <td class="colFirst"><code>org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureStoreTracker</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#toProto--">toProto</a></span>()</code>
 <div class="block">Builds
@@ -390,16 +401,16 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
  protocol buffer from current state.</div>
 </td>
 </tr>
-<tr id="i35" class="rowColor">
+<tr id="i37" class="rowColor">
 <td class="colFirst"><code>private void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#trackProcIds-long-">trackProcIds</a></span>(long&nbsp;procId)</code>&nbsp;</td>
 </tr>
-<tr id="i36" class="altColor">
+<tr id="i38" class="altColor">
 <td class="colFirst"><code>private <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#update-org.apache.hadoop.hbase.procedure2.store.BitSetNode-long-">update</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
       long&nbsp;procId)</code>&nbsp;</td>
 </tr>
-<tr id="i37" class="rowColor">
+<tr id="i39" class="rowColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#update-long-">update</a></span>(long&nbsp;procId)</code>&nbsp;</td>
 </tr>
@@ -431,7 +442,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>map</h4>
-<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/TreeMap.html?is-external=true" title="class or interface in java.util">TreeMap</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&gt; <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.43">map</a></pre>
+<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/TreeMap.html?is-external=true" title="class or interface in java.util">TreeMap</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&gt; <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.44">map</a></pre>
 </li>
 </ul>
 <a name="keepDeletes">
@@ -440,7 +451,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>keepDeletes</h4>
-<pre>private&nbsp;boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.52">keepDeletes</a></pre>
+<pre>private&nbsp;boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.53">keepDeletes</a></pre>
 <div class="block">If true, do not remove bits corresponding to deleted procedures. Note that this can result
  in huge bitmaps overtime.
  Currently, it's set to true only when building tracker state from logs during recovery. During
@@ -454,7 +465,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>partial</h4>
-<pre>boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.58">partial</a></pre>
+<pre>boolean <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.59">partial</a></pre>
 <div class="block">If true, it means tracker has incomplete information about the active/deleted procedures.
  It's set to true only when recovering from old logs. See <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#isDeleted-long-"><code>isDeleted(long)</code></a> docs to
  understand it's real use.</div>
@@ -466,7 +477,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>minModifiedProcId</h4>
-<pre>private&nbsp;long <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.60">minModifiedProcId</a></pre>
+<pre>private&nbsp;long <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.61">minModifiedProcId</a></pre>
 </li>
 </ul>
 <a name="maxModifiedProcId">
@@ -475,7 +486,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>maxModifiedProcId</h4>
-<pre>private&nbsp;long <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.61">maxModifiedProcId</a></pre>
+<pre>private&nbsp;long <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.62">maxModifiedProcId</a></pre>
 </li>
 </ul>
 </li>
@@ -492,7 +503,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>ProcedureStoreTracker</h4>
-<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.41">ProcedureStoreTracker</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.42">ProcedureStoreTracker</a>()</pre>
 </li>
 </ul>
 </li>
@@ -509,7 +520,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>resetToProto</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.65">resetToProto</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureStoreTracker&nbsp;trackerProtoBuf)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.66">resetToProto</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureStoreTracker&nbsp;trackerProtoBuf)</pre>
 </li>
 </ul>
 <a name="resetTo-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">
@@ -518,7 +529,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>resetTo</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.76">resetTo</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.77">resetTo</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</pre>
 <div class="block">Resets internal state to same as given <code>tracker</code>. Does deep copy of the bitmap.</div>
 </li>
 </ul>
@@ -528,7 +539,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>resetTo</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.88">resetTo</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.89">resetTo</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
                     boolean&nbsp;resetDelete)</pre>
 <div class="block">Resets internal state to same as given <code>tracker</code>, and change the deleted flag according
  to the modified flag if <code>resetDelete</code> is true. Does deep copy of the bitmap.
@@ -544,7 +555,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.99">insert</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.103">insert</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="insert-long:A-">
@@ -553,7 +564,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.103">insert</a>(long[]&nbsp;procIds)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.107">insert</a>(long[]&nbsp;procIds)</pre>
 </li>
 </ul>
 <a name="insert-long-long:A-">
@@ -562,7 +573,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.109">insert</a>(long&nbsp;procId,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.113">insert</a>(long&nbsp;procId,
                    long[]&nbsp;subProcIds)</pre>
 </li>
 </ul>
@@ -572,7 +583,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insert</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.116">insert</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.120">insert</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
                           long&nbsp;procId)</pre>
 </li>
 </ul>
@@ -582,7 +593,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>update</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.125">update</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.129">update</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="update-org.apache.hadoop.hbase.procedure2.store.BitSetNode-long-">
@@ -591,7 +602,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>update</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.129">update</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.133">update</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
                           long&nbsp;procId)</pre>
 </li>
 </ul>
@@ -601,7 +612,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.138">delete</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.142">delete</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="delete-long:A-">
@@ -610,7 +621,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.142">delete</a>(long[]&nbsp;procIds)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.146">delete</a>(long[]&nbsp;procIds)</pre>
 </li>
 </ul>
 <a name="delete-org.apache.hadoop.hbase.procedure2.store.BitSetNode-long-">
@@ -619,7 +630,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.150">delete</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.154">delete</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;node,
                           long&nbsp;procId)</pre>
 </li>
 </ul>
@@ -629,7 +640,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setMinMaxModifiedProcIds</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.167">setMinMaxModifiedProcIds</a>(long&nbsp;min,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.171">setMinMaxModifiedProcIds</a>(long&nbsp;min,
                                      long&nbsp;max)</pre>
 <div class="block">Will be called when restarting where we need to rebuild the ProcedureStoreTracker.</div>
 </li>
@@ -640,7 +651,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setDeleted</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.176">setDeleted</a>(long&nbsp;procId,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.180">setDeleted</a>(long&nbsp;procId,
                        boolean&nbsp;isDeleted)</pre>
 <div class="block">This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The
  <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#delete-long-"><code>delete(long)</code></a> method above assume that the <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store"><code>BitSetNode</code></a> exists, but when restart
@@ -653,7 +664,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setDeletedIfModified</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.190">setDeletedIfModified</a>(long...&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.194">setDeletedIfModified</a>(long...&nbsp;procId)</pre>
 <div class="block">Set the given bit for the procId to delete if it was modified before.
  <p/>
  This method is used to test whether a procedure wal file can be safely deleted, as if all the
@@ -661,14 +672,38 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
  then we can delete it.</div>
 </li>
 </ul>
-<a name="setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-">
+<a name="setDeleteIf-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-java.util.function.BiFunction-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>setDeleteIf</h4>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.204">setDeleteIf</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/BiFunction.html?is-external=true" title="class or interface in java.util.function">BiFunction</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;func)</pre>
+</li>
+</ul>
+<a name="setDeletedIfDeletedByThem-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>setDeletedIfDeletedByThem</h4>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.228">setDeletedIfDeletedByThem</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</pre>
+<div class="block">For the global tracker, we will use this method to build the holdingCleanupTracker, as the
+ modified flags will be cleared after rolling so we only need to test the deleted flags.</div>
+<dl>
+<dt><span class="seeLabel">See Also:</span></dt>
+<dd><a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-"><code>setDeletedIfModifiedInBoth(ProcedureStoreTracker)</code></a></dd>
+</dl>
+</li>
+</ul>
+<a name="setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">
 <!--   -->
 </a>
 <ul class="blockList">
 <li class="blockList">
 <h4>setDeletedIfModifiedInBoth</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.206">setDeletedIfModifiedInBoth</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-                                       boolean&nbsp;globalTracker)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.239">setDeletedIfModifiedInBoth</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</pre>
 <div class="block">Similar with <a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-"><code>setDeletedIfModified(long...)</code></a>, but here the <code>procId</code> are given by
  the <code>tracker</code>. If a procedure is modified by us, and also by the given <code>tracker</code>,
  then we mark it as deleted.</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/BitSetNode.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/BitSetNode.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/BitSetNode.html
index c5dece6..5b580b9 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/BitSetNode.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/BitSetNode.html
@@ -218,6 +218,20 @@
 </tr>
 </tbody>
 </table>
+<table class="useSummary" border="0" cellpadding="3" cellspacing="0" summary="Use table, listing methods, and an explanation">
+<caption><span>Method parameters in <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/package-summary.html">org.apache.hadoop.hbase.procedure2.store</a> with type arguments of type <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a></span><span class="tabEnd">&nbsp;</span></caption>
+<tr>
+<th class="colFirst" scope="col">Modifier and Type</th>
+<th class="colLast" scope="col">Method and Description</th>
+</tr>
+<tbody>
+<tr class="altColor">
+<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><span class="typeNameLabel">ProcedureStoreTracker.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeleteIf-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-java.util.function.BiFunction-">setDeleteIf</a></span>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+           <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/BiFunction.html?is-external=true" title="class or interface in java.util.function">BiFunction</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;func)</code>&nbsp;</td>
+</tr>
+</tbody>
+</table>
 <table class="useSummary" border="0" cellpadding="3" cellspacing="0" summary="Use table, listing constructors, and an explanation">
 <caption><span>Constructors in <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/package-summary.html">org.apache.hadoop.hbase.procedure2.store</a> with parameters of type <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a></span><span class="tabEnd">&nbsp;</span></caption>
 <tr>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/ProcedureStoreTracker.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/ProcedureStoreTracker.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/ProcedureStoreTracker.html
index 0ec9d05..55d8faa 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/ProcedureStoreTracker.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/class-use/ProcedureStoreTracker.html
@@ -122,12 +122,23 @@
 </tr>
 <tr class="altColor">
 <td class="colFirst"><code>void</code></td>
-<td class="colLast"><span class="typeNameLabel">ProcedureStoreTracker.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-">setDeletedIfModifiedInBoth</a></span>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-                          boolean&nbsp;globalTracker)</code>
+<td class="colLast"><span class="typeNameLabel">ProcedureStoreTracker.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfDeletedByThem-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfDeletedByThem</a></span>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</code>
+<div class="block">For the global tracker, we will use this method to build the holdingCleanupTracker, as the
+ modified flags will be cleared after rolling so we only need to test the deleted flags.</div>
+</td>
+</tr>
+<tr class="rowColor">
+<td class="colFirst"><code>void</code></td>
+<td class="colLast"><span class="typeNameLabel">ProcedureStoreTracker.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-">setDeletedIfModifiedInBoth</a></span>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)</code>
 <div class="block">Similar with <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModified-long...-"><code>setDeletedIfModified(long...)</code></a>, but here the <code>procId</code> are given by
  the <code>tracker</code>.</div>
 </td>
 </tr>
+<tr class="altColor">
+<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><span class="typeNameLabel">ProcedureStoreTracker.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeleteIf-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-java.util.function.BiFunction-">setDeleteIf</a></span>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+           <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/BiFunction.html?is-external=true" title="class or interface in java.util.function">BiFunction</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>&gt;&nbsp;func)</code>&nbsp;</td>
+</tr>
 </tbody>
 </table>
 </li>
@@ -202,7 +213,10 @@
 <td class="colFirst"><code>static void</code></td>
 <td class="colLast"><span class="typeNameLabel">ProcedureWALFormat.</span><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#load-java.util.Iterator-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.Loader-">load</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
     <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-    <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>&nbsp;</td>
+    <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.</div>
+</td>
 </tr>
 <tr class="altColor">
 <td class="colFirst"><code>private void</code></td>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
index a98c739..66fa7c4 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html
@@ -220,7 +220,10 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <td class="colFirst"><code>static void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#load-java.util.Iterator-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.Loader-">load</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
     <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-    <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>&nbsp;</td>
+    <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.</div>
+</td>
 </tr>
 <tr id="i1" class="rowColor">
 <td class="colFirst"><code>static org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry</code></td>
@@ -424,10 +427,19 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>load</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.76">load</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.87">load</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
                         <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
                         <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.
+ <p/>
+ The method in the give <code>loader</code> will be called at the end after we load all the
+ procedures and construct the hierarchy.
+ <p/>
+ And we will call the <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#resetModified--"><code>ProcedureStoreTracker.resetModified()</code></a> method for the given
+ <code>tracker</code> before returning, as it will be used to track the next proc wal file's modified
+ procedures.</div>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></code></dd>
@@ -440,7 +452,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeHeader</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.103">writeHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/OutputStream.html?is-external=true" title="class or interface in java.io">OutputStream</a>&nbsp;stream,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.114">writeHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/OutputStream.html?is-external=true" title="class or interface in java.io">OutputStream</a>&nbsp;stream,
                                org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;header)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -455,7 +467,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeTrailer</h4>
-<pre>public static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.123">writeTrailer</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
+<pre>public static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.134">writeTrailer</a>(org.apache.hadoop.fs.FSDataOutputStream&nbsp;stream,
                                 <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker)
                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -470,7 +482,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readHeader</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.141">readHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALHeader&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.152">readHeader</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
                                                                                                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -484,7 +496,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readTrailer</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALTrailer&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.166">readTrailer</a>(org.apache.hadoop.fs.FSDataInputStream&nbsp;stream,
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALTrailer&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.177">readTrailer</a>(org.apache.hadoop.fs.FSDataInputStream&nbsp;stream,
                                                                                                                 long&nbsp;startPos,
                                                                                                                 long&nbsp;size)
                                                                                                          throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -500,7 +512,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readEntry</h4>
-<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.203">readEntry</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
+<pre>public static&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.214">readEntry</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html?is-external=true" title="class or interface in java.io">InputStream</a>&nbsp;stream)
                                                                                                      throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -514,7 +526,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeEntry</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.207">writeEntry</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.218">writeEntry</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                               org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry.Type&nbsp;type,
                               <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;proc,
                               <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;[]&nbsp;subprocs)
@@ -531,7 +543,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeInsert</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.220">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.231">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;proc)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -546,7 +558,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeInsert</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.225">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.236">writeInsert</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;proc,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;[]&nbsp;subprocs)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -562,7 +574,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeUpdate</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.230">writeUpdate</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.241">writeUpdate</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;proc)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -577,7 +589,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>writeDelete</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.235">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.246">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                long&nbsp;procId)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -592,7 +604,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>writeDelete</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.243">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#line.254">writeDelete</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/util/ByteSlot.html" title="class in org.apache.hadoop.hbase.procedure2.util">ByteSlot</a>&nbsp;slot,
                                <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;proc,
                                long[]&nbsp;subprocs)
                         throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
index aaf70c9..86020ee 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html
@@ -405,7 +405,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>finish</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.142">finish</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.141">finish</a>()
             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -419,7 +419,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setDeletedIfPartial</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.153">setDeletedIfPartial</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.152">setDeletedIfPartial</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
                                  long&nbsp;procId)</pre>
 </li>
 </ul>
@@ -429,7 +429,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insertIfPartial</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.159">insertIfPartial</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.158">insertIfPartial</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
                              org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedure&nbsp;proc)</pre>
 </li>
 </ul>
@@ -439,7 +439,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>loadProcedure</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.165">loadProcedure</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.164">loadProcedure</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry,
                            org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedure&nbsp;proc)</pre>
 </li>
 </ul>
@@ -449,7 +449,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readInitEntry</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.175">readInitEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.174">readInitEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
 </li>
 </ul>
 <a name="readInsertEntry-org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry-">
@@ -458,7 +458,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readInsertEntry</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.180">readInsertEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.179">readInsertEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
 </li>
 </ul>
 <a name="readUpdateEntry-org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry-">
@@ -467,7 +467,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readUpdateEntry</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.188">readUpdateEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.187">readUpdateEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
 </li>
 </ul>
 <a name="readDeleteEntry-org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry-">
@@ -476,7 +476,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>readDeleteEntry</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.193">readDeleteEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.192">readDeleteEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureWALEntry&nbsp;entry)</pre>
 </li>
 </ul>
 <a name="deleteEntry-long-">
@@ -485,7 +485,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>deleteEntry</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.214">deleteEntry</a>(long&nbsp;procId)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.213">deleteEntry</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="isDeleted-long-">
@@ -494,7 +494,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isDeleted</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.223">isDeleted</a>(long&nbsp;procId)</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.222">isDeleted</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="isRequired-long-">
@@ -503,7 +503,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>isRequired</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.227">isRequired</a>(long&nbsp;procId)</pre>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html#line.226">isRequired</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
index 7028017..6f4f1df 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
@@ -155,7 +155,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
  will first be initialized to the oldest file's tracker(which is stored in the trailer), using the
  method <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#resetTo-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-"><code>ProcedureStoreTracker.resetTo(ProcedureStoreTracker, boolean)</code></a>, and then merge it
  with the tracker of every newer wal files, using the
- <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-boolean-"><code>ProcedureStoreTracker.setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)</code></a>.
+ <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#setDeletedIfModifiedInBoth-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-"><code>ProcedureStoreTracker.setDeletedIfModifiedInBoth(ProcedureStoreTracker)</code></a>.
  If we find out
  that all the modified procedures for the oldest wal file are modified or deleted in newer wal
  files, then we can delete it. This is because that, every time we call
@@ -1489,7 +1489,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>WALS_PATH_FILTER</h4>
-<pre>private static final&nbsp;org.apache.hadoop.fs.PathFilter <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1265">WALS_PATH_FILTER</a></pre>
+<pre>private static final&nbsp;org.apache.hadoop.fs.PathFilter <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1264">WALS_PATH_FILTER</a></pre>
 </li>
 </ul>
 <a name="FILE_STATUS_ID_COMPARATOR">
@@ -1498,7 +1498,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockListLast">
 <li class="blockList">
 <h4>FILE_STATUS_ID_COMPARATOR</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;org.apache.hadoop.fs.FileStatus&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1273">FILE_STATUS_ID_COMPARATOR</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Comparator.html?is-external=true" title="class or interface in java.util">Comparator</a>&lt;org.apache.hadoop.fs.FileStatus&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1272">FILE_STATUS_ID_COMPARATOR</a></pre>
 </li>
 </ul>
 </li>
@@ -2084,7 +2084,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeAllLogs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1203">removeAllLogs</a>(long&nbsp;lastLogId,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1202">removeAllLogs</a>(long&nbsp;lastLogId,
                            <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;why)</pre>
 <div class="block">Remove all logs with logId <= <code>lastLogId</code>.</div>
 </li>
@@ -2095,7 +2095,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>removeLogFile</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1225">removeLogFile</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;log,
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1224">removeLogFile</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;log,
                               org.apache.hadoop.fs.Path&nbsp;walArchiveDir)</pre>
 </li>
 </ul>
@@ -2105,7 +2105,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getWALDir</h4>
-<pre>public&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1242">getWALDir</a>()</pre>
+<pre>public&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1241">getWALDir</a>()</pre>
 </li>
 </ul>
 <a name="getWalArchiveDir--">
@@ -2114,7 +2114,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getWalArchiveDir</h4>
-<pre>org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1247">getWalArchiveDir</a>()</pre>
+<pre>org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1246">getWalArchiveDir</a>()</pre>
 </li>
 </ul>
 <a name="getFileSystem--">
@@ -2123,7 +2123,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getFileSystem</h4>
-<pre>public&nbsp;org.apache.hadoop.fs.FileSystem&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1251">getFileSystem</a>()</pre>
+<pre>public&nbsp;org.apache.hadoop.fs.FileSystem&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1250">getFileSystem</a>()</pre>
 </li>
 </ul>
 <a name="getLogFilePath-long-">
@@ -2132,7 +2132,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogFilePath</h4>
-<pre>protected&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1255">getLogFilePath</a>(long&nbsp;logId)
+<pre>protected&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1254">getLogFilePath</a>(long&nbsp;logId)
                                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2146,7 +2146,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogIdFromName</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1259">getLogIdFromName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1258">getLogIdFromName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 </li>
 </ul>
 <a name="getLogFiles--">
@@ -2155,7 +2155,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getLogFiles</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.FileStatus[]&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1283">getLogFiles</a>()
+<pre>private&nbsp;org.apache.hadoop.fs.FileStatus[]&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1282">getLogFiles</a>()
                                                throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -2169,7 +2169,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>getMaxLogId</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1299">getMaxLogId</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1298">getMaxLogId</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)</pre>
 <div class="block">Make sure that the file set are gotten by calling <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#getLogFiles--"><code>getLogFiles()</code></a>, where we will sort
  the file set by log id.</div>
 <dl>
@@ -2184,7 +2184,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>initOldLogs</h4>
-<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1311">initOldLogs</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)
+<pre>private&nbsp;long&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1310">initOldLogs</a>(org.apache.hadoop.fs.FileStatus[]&nbsp;logFiles)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Make sure that the file set are gotten by calling <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#getLogFiles--"><code>getLogFiles()</code></a>, where we will sort
  the file set by log id.</div>
@@ -2202,7 +2202,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>initTrackerFromOldLogs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1337">initTrackerFromOldLogs</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1336">initTrackerFromOldLogs</a>()</pre>
 <div class="block">If last log's tracker is not null, use it as <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#storeTracker"><code>storeTracker</code></a>. Otherwise, set storeTracker
  as partial, and let <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal"><code>ProcedureWALFormatReader</code></a> rebuild it using entries in the log.</div>
 </li>
@@ -2213,7 +2213,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockList">
 <li class="blockList">
 <h4>initOldLog</h4>
-<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1353">initOldLog</a>(org.apache.hadoop.fs.FileStatus&nbsp;logFile,
+<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1352">initOldLog</a>(org.apache.hadoop.fs.FileStatus&nbsp;logFile,
                                     org.apache.hadoop.fs.Path&nbsp;walArchiveDir)
                              throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Loads given log file and it's tracker.</div>
@@ -2229,7 +2229,7 @@ extends <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/P
 <ul class="blockListLast">
 <li class="blockList">
 <h4>main</h4>
-<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1391">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;args)
+<pre>public static&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html#line.1390">main</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;args)
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Parses a directory of WALs building up ProcedureState.
  For testing parse and profiling.</div>


[04/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
index 9b964f6..98ef11a 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.html
@@ -105,7 +105,7 @@
 <span class="sourceLineNo">097</span> * will first be initialized to the oldest file's tracker(which is stored in the trailer), using the<a name="line.97"></a>
 <span class="sourceLineNo">098</span> * method {@link ProcedureStoreTracker#resetTo(ProcedureStoreTracker, boolean)}, and then merge it<a name="line.98"></a>
 <span class="sourceLineNo">099</span> * with the tracker of every newer wal files, using the<a name="line.99"></a>
-<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)}.<a name="line.100"></a>
+<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker)}.<a name="line.100"></a>
 <span class="sourceLineNo">101</span> * If we find out<a name="line.101"></a>
 <span class="sourceLineNo">102</span> * that all the modified procedures for the oldest wal file are modified or deleted in newer wal<a name="line.102"></a>
 <span class="sourceLineNo">103</span> * files, then we can delete it. This is because that, every time we call<a name="line.103"></a>
@@ -1181,244 +1181,243 @@
 <span class="sourceLineNo">1173</span>    }<a name="line.1173"></a>
 <span class="sourceLineNo">1174</span><a name="line.1174"></a>
 <span class="sourceLineNo">1175</span>    // compute the holding tracker.<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>    //  - the first WAL is used for the 'updates'<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span>    //  - the global tracker is passed in first to decide which procedures are not<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>    //    exist anymore, so we can mark them as deleted in holdingCleanupTracker.<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>    //    Only global tracker have the whole picture here.<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>    //  - the other WALs are scanned to remove procs already updated in a newer wal.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>    //    If it is updated in a newer wal, we can mark it as delelted in holdingCleanupTracker<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span>    //    But, we can not delete it if it was shown deleted in the newer wal, as said<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>    //    above.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span>    // TODO: exit early if holdingCleanupTracker.isEmpty()<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>    //Passing in the global tracker, we can delete the procedures not in the global<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>    //tracker, because they are deleted in the later logs<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    holdingCleanupTracker.setDeletedIfModifiedInBoth(storeTracker, true);<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    for (int i = 1, size = logs.size() - 1; i &lt; size; ++i) {<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>      // Set deleteIfNotExists to false since a single log's tracker is passed in.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>      // Since a specific procedure may not show up in the log at all(not executed or<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span>      // updated during the time), we can not delete the procedure just because this log<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>      // don't have the info of the procedure. We can delete the procedure only if<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>      // in this log's tracker, it was cleanly showed that the procedure is modified or deleted<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>      // in the corresponding BitSetNode.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(logs.get(i).getTracker(), false);<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span>    }<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>  }<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span><a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>  /**<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>   */<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    if (logs.size() &lt;= 1) {<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>      return;<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    }<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span><a name="line.1207"></a>
-<span class="sourceLineNo">1208</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span><a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>    boolean removed = false;<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    while (logs.size() &gt; 1) {<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>        break;<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>      }<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>      removeLogFile(log, walArchiveDir);<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>      removed = true;<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    }<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span><a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>    if (removed) {<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>      buildHoldingCleanupTracker();<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>    }<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>  }<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span><a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>    try {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span>      LOG.trace("Removing log={}", log);<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>      log.removeFile(walArchiveDir);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>      logs.remove(log);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>    } catch (IOException e) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>      return false;<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    }<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>    return true;<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>  }<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span><a name="line.1238"></a>
-<span class="sourceLineNo">1239</span>  // ==========================================================================<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  //  FileSystem Log Files helpers<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>  // ==========================================================================<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>  public Path getWALDir() {<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>    return this.walDir;<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span>  }<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span><a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>  @VisibleForTesting<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span>  Path getWalArchiveDir() {<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    return this.walArchiveDir;<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>  }<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span><a name="line.1250"></a>
-<span class="sourceLineNo">1251</span>  public FileSystem getFileSystem() {<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    return this.fs;<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>  }<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span><a name="line.1254"></a>
-<span class="sourceLineNo">1255</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  private static long getLogIdFromName(final String name) {<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>    int end = name.lastIndexOf(".log");<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>    return Long.parseLong(name.substring(start, end));<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span>  }<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span><a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    @Override<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>    public boolean accept(Path path) {<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span>      String name = path.getName();<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>    }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>  };<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span><a name="line.1272"></a>
-<span class="sourceLineNo">1273</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>    @Override<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>      return Long.compare(aId, bId);<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>    }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>  };<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span><a name="line.1282"></a>
-<span class="sourceLineNo">1283</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>    try {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>      return files;<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>    } catch (FileNotFoundException e) {<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>      return null;<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>    }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>  }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span><a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>  /**<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>   * the file set by log id.<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   * @return Max-LogID of the specified log file set<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>   */<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>      return 0L;<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>    }<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>  }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span><a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>  /**<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   * the file set by log id.<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   * @return Max-LogID of the specified log file set<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   */<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>      return 0L;<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>    }<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>    long maxLogId = 0;<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      final Path logPath = logFiles[i].getPath();<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>      if (!isRunning()) {<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>        throw new IOException("wal aborting");<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span>      }<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span><a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>      if (log != null) {<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>        this.logs.add(log);<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>      }<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    }<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span>    initTrackerFromOldLogs();<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    return maxLogId;<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>  }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span>  /**<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>   */<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>  private void initTrackerFromOldLogs() {<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span>      return;<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>    ProcedureWALFile log = logs.getLast();<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (!log.getTracker().isPartial()) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      storeTracker.resetTo(log.getTracker());<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>    } else {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>      storeTracker.reset();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      storeTracker.setPartialFlag(true);<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>    }<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>  }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span><a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>  /**<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>   * Loads given log file and it's tracker.<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>   */<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>      throws IOException {<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span>    if (logFile.getLen() == 0) {<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span>      log.removeFile(walArchiveDir);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>      return null;<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>    }<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span>    try {<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>      log.open();<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>      log.removeFile(walArchiveDir);<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      return null;<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span>    } catch (IOException e) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      LOG.error(msg, e);<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span>      throw new IOException(msg, e);<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span>    }<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span><a name="line.1373"></a>
-<span class="sourceLineNo">1374</span>    try {<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>      log.readTracker();<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>    } catch (IOException e) {<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>      log.getTracker().reset();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>      log.getTracker().setPartialFlag(true);<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span>    }<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span><a name="line.1381"></a>
-<span class="sourceLineNo">1382</span>    log.close();<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span>    return log;<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span>  }<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span><a name="line.1385"></a>
-<span class="sourceLineNo">1386</span>  /**<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span>   * For testing parse and profiling.<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span>   */<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span>  public static void main(String [] args) throws IOException {<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span>    if (args == null || args.length != 1) {<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span>      System.exit(-1);<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span>    }<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span>        @Override<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span>          // no-op<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span>        }<a name="line.1403"></a>
-<span class="sourceLineNo">1404</span>      });<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span>    try {<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span>      store.start(16);<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span>      pe.init(1, true);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span>    } finally {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span>      store.stop(true);<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span>    }<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span>  }<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span>}<a name="line.1413"></a>
+<span class="sourceLineNo">1176</span>    // - the first WAL is used for the 'updates'<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    // - the global tracker will be used to determine whether a procedure has been deleted<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    // - other trackers will be used to determine whether a procedure has been updated, as a deleted<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>    // procedure can always be detected by checking the global tracker, we can save the deleted<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span>    // checks when applying other trackers<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span>    holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span>    // the logs is a linked list, so avoid calling get(index) on it.<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span>    Iterator&lt;ProcedureWALFile&gt; iter = logs.iterator();<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span>    // skip the tracker for the first file when creating the iterator.<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>    iter.next();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>    ProcedureStoreTracker tracker = iter.next().getTracker();<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>    // testing iter.hasNext after calling iter.next to skip applying the tracker for last file,<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>    // which is just the storeTracker above.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>    while (iter.hasNext()) {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker);<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>      if (holdingCleanupTracker.isEmpty()) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>        break;<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>      }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span>      iter.next();<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>    }<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span>  }<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span><a name="line.1198"></a>
+<span class="sourceLineNo">1199</span>  /**<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   */<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>    if (logs.size() &lt;= 1) {<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>      return;<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>    }<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span><a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span><a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    boolean removed = false;<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>    while (logs.size() &gt; 1) {<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>        break;<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>      }<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>      removeLogFile(log, walArchiveDir);<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span>      removed = true;<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>    }<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span><a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>    if (removed) {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>      buildHoldingCleanupTracker();<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    }<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>  }<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span><a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>    try {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>      LOG.trace("Removing log={}", log);<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      log.removeFile(walArchiveDir);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>      logs.remove(log);<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>    } catch (IOException e) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>      return false;<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    }<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>    return true;<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span>  }<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span><a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>  // ==========================================================================<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>  //  FileSystem Log Files helpers<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>  // ==========================================================================<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  public Path getWALDir() {<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>    return this.walDir;<a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span><a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>  @VisibleForTesting<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  Path getWalArchiveDir() {<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span>    return this.walArchiveDir;<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span>  }<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span><a name="line.1249"></a>
+<span class="sourceLineNo">1250</span>  public FileSystem getFileSystem() {<a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    return this.fs;<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>  }<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span><a name="line.1253"></a>
+<span class="sourceLineNo">1254</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>  }<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span><a name="line.1257"></a>
+<span class="sourceLineNo">1258</span>  private static long getLogIdFromName(final String name) {<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span>    int end = name.lastIndexOf(".log");<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span>    return Long.parseLong(name.substring(start, end));<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span><a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>    @Override<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span>    public boolean accept(Path path) {<a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>      String name = path.getName();<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    }<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  };<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    @Override<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>      return Long.compare(aId, bId);<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    }<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  };<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>    try {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span>      return files;<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>    } catch (FileNotFoundException e) {<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>      return null;<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>    }<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>  }<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span><a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>  /**<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   * the file set by log id.<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   * @return Max-LogID of the specified log file set<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>   */<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>      return 0L;<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>    }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>  }<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span><a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>  /**<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   * the file set by log id.<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   * @return Max-LogID of the specified log file set<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   */<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>      return 0L;<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>    }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span>    long maxLogId = 0;<a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>      final Path logPath = logFiles[i].getPath();<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>      if (!isRunning()) {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>        throw new IOException("wal aborting");<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      }<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span><a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span>      if (log != null) {<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>        this.logs.add(log);<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>      }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    }<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    initTrackerFromOldLogs();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    return maxLogId;<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>  }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span><a name="line.1331"></a>
+<span class="sourceLineNo">1332</span>  /**<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>   */<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span>  private void initTrackerFromOldLogs() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>      return;<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>    }<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    ProcedureWALFile log = logs.getLast();<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>    if (!log.getTracker().isPartial()) {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span>      storeTracker.resetTo(log.getTracker());<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>    } else {<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>      storeTracker.reset();<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>      storeTracker.setPartialFlag(true);<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>    }<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>  }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>  /**<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>   * Loads given log file and it's tracker.<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>   */<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>      throws IOException {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>    if (logFile.getLen() == 0) {<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>      log.removeFile(walArchiveDir);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>      return null;<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span>    }<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span>    try {<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>      log.open();<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span>      log.removeFile(walArchiveDir);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>      return null;<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>    } catch (IOException e) {<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span>      LOG.error(msg, e);<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      throw new IOException(msg, e);<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span><a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>    try {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>      log.readTracker();<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span>    } catch (IOException e) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>      log.getTracker().reset();<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span>      log.getTracker().setPartialFlag(true);<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span>    }<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span><a name="line.1380"></a>
+<span class="sourceLineNo">1381</span>    log.close();<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span>    return log;<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span>  }<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span><a name="line.1384"></a>
+<span class="sourceLineNo">1385</span>  /**<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span>   * For testing parse and profiling.<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span>   */<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span>  public static void main(String [] args) throws IOException {<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span>    if (args == null || args.length != 1) {<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span>      System.exit(-1);<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span>    }<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span>        @Override<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span>          // no-op<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span>        }<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span>      });<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span>    try {<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span>      store.start(16);<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span>      pe.init(1, true);<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span>    } finally {<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span>      store.stop(true);<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span>    }<a name="line.1410"></a>
+<span class="sourceLineNo">1411</span>  }<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span>}<a name="line.1412"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html
index 8ef0ff5..9545561 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html
@@ -153,158 +153,161 @@
 <span class="sourceLineNo">145</span><a name="line.145"></a>
 <span class="sourceLineNo">146</span>  private void collectStackId(Entry entry, Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc,<a name="line.146"></a>
 <span class="sourceLineNo">147</span>      MutableInt maxStackId) {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      int stackId = entry.proc.getStackId(i);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>        maxStackId.setValue(stackId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    }<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.155"></a>
-<span class="sourceLineNo">156</span>  }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    for (Entry e : entry.subProcs) {<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    }<a name="line.164"></a>
-<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    for (Entry e : entry.subProcs) {<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.176"></a>
-<span class="sourceLineNo">177</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  // orphan procedures.<a name="line.178"></a>
-<span class="sourceLineNo">179</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    if (isFinished(rootEntry.proc)) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.183"></a>
-<span class="sourceLineNo">184</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.184"></a>
-<span class="sourceLineNo">185</span>      } else {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      }<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      return;<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.192"></a>
-<span class="sourceLineNo">193</span>    // the stack ids should start from 0 and increase by one every time<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    boolean valid = true;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      if (entries == null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          rootEntry);<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        valid = false;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      } else if (entries.size() &gt; 1) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.202"></a>
-<span class="sourceLineNo">203</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.203"></a>
-<span class="sourceLineNo">204</span>        valid = false;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      }<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    }<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    if (valid) {<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    } else {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
-<span class="sourceLineNo">213</span><a name="line.213"></a>
-<span class="sourceLineNo">214</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>    procMap.values().forEach(entry -&gt; {<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>    });<a name="line.218"></a>
-<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>  private static final class Iter implements ProcedureIterator {<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>    private ProtoAndProc current;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>      this.procs = procs;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      reset();<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    }<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span>    @Override<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    public void reset() {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      iter = procs.iterator();<a name="line.236"></a>
-<span class="sourceLineNo">237</span>      if (iter.hasNext()) {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        current = iter.next();<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      } else {<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        current = null;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>    }<a name="line.242"></a>
-<span class="sourceLineNo">243</span><a name="line.243"></a>
-<span class="sourceLineNo">244</span>    @Override<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    public boolean hasNext() {<a name="line.245"></a>
-<span class="sourceLineNo">246</span>      return current != null;<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    }<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span>    private void checkNext() {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      if (!hasNext()) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        throw new NoSuchElementException();<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span><a name="line.254"></a>
-<span class="sourceLineNo">255</span>    @Override<a name="line.255"></a>
-<span class="sourceLineNo">256</span>    public boolean isNextFinished() {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>      checkNext();<a name="line.257"></a>
-<span class="sourceLineNo">258</span>      return isFinished(current.proto);<a name="line.258"></a>
-<span class="sourceLineNo">259</span>    }<a name="line.259"></a>
-<span class="sourceLineNo">260</span><a name="line.260"></a>
-<span class="sourceLineNo">261</span>    private void moveToNext() {<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      if (iter.hasNext()) {<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        current = iter.next();<a name="line.263"></a>
-<span class="sourceLineNo">264</span>      } else {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>        current = null;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>    }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span>    @Override<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    public void skipNext() {<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      checkNext();<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      moveToNext();<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span>    @Override<a name="line.275"></a>
-<span class="sourceLineNo">276</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      checkNext();<a name="line.277"></a>
-<span class="sourceLineNo">278</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.278"></a>
-<span class="sourceLineNo">279</span>      moveToNext();<a name="line.279"></a>
-<span class="sourceLineNo">280</span>      return proc;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>    }<a name="line.281"></a>
-<span class="sourceLineNo">282</span>  }<a name="line.282"></a>
-<span class="sourceLineNo">283</span><a name="line.283"></a>
-<span class="sourceLineNo">284</span>  public ProcedureIterator getValidProcs() {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return new Iter(validProcs);<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    return new Iter(corruptedProcs);<a name="line.289"></a>
-<span class="sourceLineNo">290</span>  }<a name="line.290"></a>
-<span class="sourceLineNo">291</span><a name="line.291"></a>
-<span class="sourceLineNo">292</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    }<a name="line.296"></a>
-<span class="sourceLineNo">297</span>    return new WALProcedureTree(procMap);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>  }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>}<a name="line.299"></a>
+<span class="sourceLineNo">148</span>    if (LOG.isDebugEnabled()) {<a name="line.148"></a>
+<span class="sourceLineNo">149</span>      LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList());<a name="line.149"></a>
+<span class="sourceLineNo">150</span>    }<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      int stackId = entry.proc.getStackId(i);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>        maxStackId.setValue(stackId);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      }<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.158"></a>
+<span class="sourceLineNo">159</span>  }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    for (Entry e : entry.subProcs) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
+<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    for (Entry e : entry.subProcs) {<a name="line.173"></a>
+<span class="sourceLineNo">174</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
+<span class="sourceLineNo">176</span>  }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  // orphan procedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    if (isFinished(rootEntry.proc)) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.185"></a>
+<span class="sourceLineNo">186</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      } else {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>      }<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      return;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    }<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    // the stack ids should start from 0 and increase by one every time<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    boolean valid = true;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      if (entries == null) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.201"></a>
+<span class="sourceLineNo">202</span>          rootEntry);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        valid = false;<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      } else if (entries.size() &gt; 1) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.205"></a>
+<span class="sourceLineNo">206</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.206"></a>
+<span class="sourceLineNo">207</span>        valid = false;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      }<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    if (valid) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    } else {<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.213"></a>
+<span class="sourceLineNo">214</span>    }<a name="line.214"></a>
+<span class="sourceLineNo">215</span>  }<a name="line.215"></a>
+<span class="sourceLineNo">216</span><a name="line.216"></a>
+<span class="sourceLineNo">217</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    procMap.values().forEach(entry -&gt; {<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    });<a name="line.221"></a>
+<span class="sourceLineNo">222</span>  }<a name="line.222"></a>
+<span class="sourceLineNo">223</span><a name="line.223"></a>
+<span class="sourceLineNo">224</span>  private static final class Iter implements ProcedureIterator {<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.226"></a>
+<span class="sourceLineNo">227</span><a name="line.227"></a>
+<span class="sourceLineNo">228</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>    private ProtoAndProc current;<a name="line.230"></a>
+<span class="sourceLineNo">231</span><a name="line.231"></a>
+<span class="sourceLineNo">232</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      this.procs = procs;<a name="line.233"></a>
+<span class="sourceLineNo">234</span>      reset();<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    }<a name="line.235"></a>
+<span class="sourceLineNo">236</span><a name="line.236"></a>
+<span class="sourceLineNo">237</span>    @Override<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    public void reset() {<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      iter = procs.iterator();<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (iter.hasNext()) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        current = iter.next();<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      } else {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>        current = null;<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      }<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span><a name="line.246"></a>
+<span class="sourceLineNo">247</span>    @Override<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    public boolean hasNext() {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      return current != null;<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span>    private void checkNext() {<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      if (!hasNext()) {<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        throw new NoSuchElementException();<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      }<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    }<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span>    @Override<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    public boolean isNextFinished() {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      checkNext();<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      return isFinished(current.proto);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    private void moveToNext() {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      if (iter.hasNext()) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>        current = iter.next();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      } else {<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        current = null;<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>    }<a name="line.270"></a>
+<span class="sourceLineNo">271</span><a name="line.271"></a>
+<span class="sourceLineNo">272</span>    @Override<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    public void skipNext() {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      checkNext();<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      moveToNext();<a name="line.275"></a>
+<span class="sourceLineNo">276</span>    }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span>    @Override<a name="line.278"></a>
+<span class="sourceLineNo">279</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      checkNext();<a name="line.280"></a>
+<span class="sourceLineNo">281</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.281"></a>
+<span class="sourceLineNo">282</span>      moveToNext();<a name="line.282"></a>
+<span class="sourceLineNo">283</span>      return proc;<a name="line.283"></a>
+<span class="sourceLineNo">284</span>    }<a name="line.284"></a>
+<span class="sourceLineNo">285</span>  }<a name="line.285"></a>
+<span class="sourceLineNo">286</span><a name="line.286"></a>
+<span class="sourceLineNo">287</span>  public ProcedureIterator getValidProcs() {<a name="line.287"></a>
+<span class="sourceLineNo">288</span>    return new Iter(validProcs);<a name="line.288"></a>
+<span class="sourceLineNo">289</span>  }<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    return new Iter(corruptedProcs);<a name="line.292"></a>
+<span class="sourceLineNo">293</span>  }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.297"></a>
+<span class="sourceLineNo">298</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.298"></a>
+<span class="sourceLineNo">299</span>    }<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    return new WALProcedureTree(procMap);<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span>}<a name="line.302"></a>
 
 
 


[46/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
index abd92a9..1799704 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
@@ -198,8 +198,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.LocalityType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.Type.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.Action.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.LocalityType</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
index 06c892b..c4ab76b 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
@@ -347,10 +347,10 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/RegionState.State.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">RegionState.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MasterRpcServices.BalanceSwitchMode.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MasterRpcServices.BalanceSwitchMode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.ResubmitDirective</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/RegionState.State.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">RegionState.State</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MetricsMasterSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MetricsMasterSourceFactoryImpl.FactoryStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MasterRpcServices.BalanceSwitchMode.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MasterRpcServices.BalanceSwitchMode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.TerminationStatus</span></a></li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
index 8efeb79..3d02c9f 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
@@ -215,9 +215,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.TableOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">TableProcedureInterface.TableOperationType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.PeerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">PeerProcedureInterface.PeerOperationType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/MetaProcedureInterface.MetaOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">MetaProcedureInterface.MetaOperationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.TableOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">TableProcedureInterface.TableOperationType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.ServerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">ServerProcedureInterface.ServerOperationType</span></a></li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/package-tree.html b/devapidocs/org/apache/hadoop/hbase/package-tree.html
index 4c03bae..cfacbae 100644
--- a/devapidocs/org/apache/hadoop/hbase/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/package-tree.html
@@ -437,19 +437,19 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Size.Unit.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Size.Unit</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CellBuilderType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CellBuilderType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeepDeletedCells.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeepDeletedCells</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Cell.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Cell.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HealthChecker.HealthCheckerExitStatus.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HealthChecker.HealthCheckerExitStatus</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompatibilitySingletonFactory.SingletonStorage.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompatibilitySingletonFactory.SingletonStorage</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeepDeletedCells.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeepDeletedCells</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CellBuilderType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CellBuilderType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MetaTableAccessor.QueryType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MetaTableAccessor.QueryType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Size.Unit.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Size.Unit</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HealthChecker.HealthCheckerExitStatus.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HealthChecker.HealthCheckerExitStatus</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeyValue.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeyValue.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterMetrics.Option</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Coprocessor.State.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Coprocessor.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeyValue.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeyValue.Type</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html b/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
index a5f74c7..ea70060 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>private static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.45">RootProcedureState.State</a>
+<pre>private static enum <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.49">RootProcedureState.State</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a>&gt;</pre>
 </li>
 </ul>
@@ -213,7 +213,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>RUNNING</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.46">RUNNING</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.50">RUNNING</a></pre>
 </li>
 </ul>
 <a name="FAILED">
@@ -222,7 +222,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>FAILED</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.47">FAILED</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.51">FAILED</a></pre>
 </li>
 </ul>
 <a name="ROLLINGBACK">
@@ -231,7 +231,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>ROLLINGBACK</h4>
-<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.48">ROLLINGBACK</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html#line.52">ROLLINGBACK</a></pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.html b/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
index f905785..32f3ea6 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
@@ -111,7 +111,7 @@ var activeTableTab = "activeTableTab";
 <br>
 <pre>@InterfaceAudience.Private
  @InterfaceStability.Evolving
-class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.43">RootProcedureState</a>&lt;TEnvironment&gt;
+class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.45">RootProcedureState</a>&lt;TEnvironment&gt;
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".
  A "Root Procedure" is a Procedure without parent, each subprocedure will be
@@ -160,18 +160,22 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <th class="colLast" scope="col">Field and Description</th>
 </tr>
 <tr class="altColor">
+<td class="colFirst"><code>private static org.slf4j.Logger</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html#LOG">LOG</a></span></code>&nbsp;</td>
+</tr>
+<tr class="rowColor">
 <td class="colFirst"><code>private int</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html#running">running</a></span></code>&nbsp;</td>
 </tr>
-<tr class="rowColor">
+<tr class="altColor">
 <td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html#state">state</a></span></code>&nbsp;</td>
 </tr>
-<tr class="altColor">
+<tr class="rowColor">
 <td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt;</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html#subprocs">subprocs</a></span></code>&nbsp;</td>
 </tr>
-<tr class="rowColor">
+<tr class="altColor">
 <td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt;</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html#subprocStack">subprocStack</a></span></code>&nbsp;</td>
 </tr>
@@ -300,13 +304,22 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <!--   -->
 </a>
 <h3>Field Detail</h3>
+<a name="LOG">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>LOG</h4>
+<pre>private static final&nbsp;org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.47">LOG</a></pre>
+</li>
+</ul>
 <a name="subprocs">
 <!--   -->
 </a>
 <ul class="blockList">
 <li class="blockList">
 <h4>subprocs</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.51">subprocs</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.55">subprocs</a></pre>
 </li>
 </ul>
 <a name="subprocStack">
@@ -315,7 +328,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>subprocStack</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.52">subprocStack</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.56">subprocStack</a></pre>
 </li>
 </ul>
 <a name="state">
@@ -324,7 +337,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>state</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.53">state</a></pre>
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2">RootProcedureState.State</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.57">state</a></pre>
 </li>
 </ul>
 <a name="running">
@@ -333,7 +346,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>running</h4>
-<pre>private&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.54">running</a></pre>
+<pre>private&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.58">running</a></pre>
 </li>
 </ul>
 </li>
@@ -350,7 +363,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RootProcedureState</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.43">RootProcedureState</a>()</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.45">RootProcedureState</a>()</pre>
 </li>
 </ul>
 </li>
@@ -367,7 +380,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isFailed</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.56">isFailed</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.60">isFailed</a>()</pre>
 </li>
 </ul>
 <a name="isRollingback--">
@@ -376,7 +389,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isRollingback</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.67">isRollingback</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.71">isRollingback</a>()</pre>
 </li>
 </ul>
 <a name="setRollback--">
@@ -385,7 +398,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setRollback</h4>
-<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.74">setRollback</a>()</pre>
+<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.78">setRollback</a>()</pre>
 <div class="block">Called by the ProcedureExecutor to mark rollback execution</div>
 </li>
 </ul>
@@ -395,7 +408,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>unsetRollback</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.85">unsetRollback</a>()</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.89">unsetRollback</a>()</pre>
 <div class="block">Called by the ProcedureExecutor to mark rollback execution</div>
 </li>
 </ul>
@@ -405,7 +418,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getSubprocedureIds</h4>
-<pre>protected&nbsp;long[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.90">getSubprocedureIds</a>()</pre>
+<pre>protected&nbsp;long[]&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.94">getSubprocedureIds</a>()</pre>
 </li>
 </ul>
 <a name="getSubproceduresStack--">
@@ -414,7 +427,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getSubproceduresStack</h4>
-<pre>protected&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.97">getSubproceduresStack</a>()</pre>
+<pre>protected&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.101">getSubproceduresStack</a>()</pre>
 </li>
 </ul>
 <a name="getException--">
@@ -423,7 +436,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getException</h4>
-<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RemoteProcedureException.html" title="class in org.apache.hadoop.hbase.procedure2">RemoteProcedureException</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.101">getException</a>()</pre>
+<pre>protected&nbsp;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RemoteProcedureException.html" title="class in org.apache.hadoop.hbase.procedure2">RemoteProcedureException</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.105">getException</a>()</pre>
 </li>
 </ul>
 <a name="acquire-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -432,7 +445,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>acquire</h4>
-<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.115">acquire</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
+<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.119">acquire</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
 <div class="block">Called by the ProcedureExecutor to mark the procedure step as running.</div>
 </li>
 </ul>
@@ -442,7 +455,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>release</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.127">release</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.131">release</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
 <div class="block">Called by the ProcedureExecutor to mark the procedure step as finished.</div>
 </li>
 </ul>
@@ -452,7 +465,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>abort</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.131">abort</a>()</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.135">abort</a>()</pre>
 </li>
 </ul>
 <a name="addRollbackStep-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -461,7 +474,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>addRollbackStep</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.141">addRollbackStep</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.145">addRollbackStep</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
 <div class="block">Called by the ProcedureExecutor after the procedure step is completed,
  to add the step to the rollback list (or procedure stack)</div>
 </li>
@@ -472,7 +485,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>addSubProcedure</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.152">addSubProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.157">addSubProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
 </li>
 </ul>
 <a name="loadStack-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -481,7 +494,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>loadStack</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.169">loadStack</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.174">loadStack</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="type parameter in RootProcedureState">TEnvironment</a>&gt;&nbsp;proc)</pre>
 <div class="block">Called on store load by the ProcedureExecutor to load part of the stack.
 
  Each procedure has its own stack-positions. Which means we have to write
@@ -495,7 +508,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>isValid</h4>
-<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.195">isValid</a>()</pre>
+<pre>protected&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html#line.200">isValid</a>()</pre>
 <div class="block">Called on store load by the ProcedureExecutor to validate the procedure stack.</div>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
index bec2a9f..967461e 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
@@ -216,11 +216,11 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResourceType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockedResourceType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">RootProcedureState.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">Procedure.LockState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/StateMachineProcedure.Flow.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">StateMachineProcedure.Flow</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResourceType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockedResourceType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/StateMachineProcedure.Flow.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">StateMachineProcedure.Flow</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">Procedure.LockState</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
index 3e087e9..8978238 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
@@ -516,7 +516,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>insertOrUpdate</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.151">insertOrUpdate</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.153">insertOrUpdate</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="delete-long-">
@@ -525,7 +525,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>delete</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.155">delete</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.157">delete</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="getStart--">
@@ -534,7 +534,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getStart</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.159">getStart</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.161">getStart</a>()</pre>
 </li>
 </ul>
 <a name="getEnd--">
@@ -543,7 +543,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getEnd</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.163">getEnd</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.165">getEnd</a>()</pre>
 </li>
 </ul>
 <a name="contains-long-">
@@ -552,7 +552,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>contains</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.167">contains</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.169">contains</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="isDeleted-long-">
@@ -561,7 +561,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isDeleted</h4>
-<pre>public&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.171">isDeleted</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.173">isDeleted</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="isModified-long-">
@@ -570,7 +570,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isModified</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.180">isModified</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.182">isModified</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="isAllModified--">
@@ -579,7 +579,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isAllModified</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.192">isAllModified</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.194">isAllModified</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>true, if all the procedures has been modified.</dd>
@@ -592,7 +592,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getActiveProcIds</h4>
-<pre>public&nbsp;long[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.205">getActiveProcIds</a>()</pre>
+<pre>public&nbsp;long[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.207">getActiveProcIds</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>all the active procedure ids in this bit set.</dd>
@@ -605,7 +605,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isEmpty</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.226">isEmpty</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.228">isEmpty</a>()</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>true, if there are no active procedures in this BitSetNode, else false.</dd>
@@ -618,7 +618,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>resetModified</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.236">resetModified</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.238">resetModified</a>()</pre>
 </li>
 </ul>
 <a name="unsetPartialFlag--">
@@ -627,7 +627,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>unsetPartialFlag</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.240">unsetPartialFlag</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.242">unsetPartialFlag</a>()</pre>
 </li>
 </ul>
 <a name="convert--">
@@ -636,7 +636,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>convert</h4>
-<pre>public&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureStoreTracker.TrackerNode&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.256">convert</a>()</pre>
+<pre>public&nbsp;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureStoreTracker.TrackerNode&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.258">convert</a>()</pre>
 <div class="block">Convert to
  org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureStoreTracker.TrackerNode
  protobuf.</div>
@@ -648,7 +648,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>canGrow</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.270">canGrow</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.272">canGrow</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="canMerge-org.apache.hadoop.hbase.procedure2.store.BitSetNode-">
@@ -657,7 +657,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>canMerge</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.274">canMerge</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;rightNode)</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.276">canMerge</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;rightNode)</pre>
 </li>
 </ul>
 <a name="grow-long-">
@@ -666,7 +666,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>grow</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.280">grow</a>(long&nbsp;procId)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.282">grow</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="merge-org.apache.hadoop.hbase.procedure2.store.BitSetNode-">
@@ -675,7 +675,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>merge</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.314">merge</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;rightNode)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.316">merge</a>(<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/BitSetNode.html" title="class in org.apache.hadoop.hbase.procedure2.store">BitSetNode</a>&nbsp;rightNode)</pre>
 </li>
 </ul>
 <a name="toString--">
@@ -684,7 +684,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>toString</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.339">toString</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.341">toString</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code>&nbsp;in class&nbsp;<code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></code></dd>
@@ -697,7 +697,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getActiveMinProcId</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.346">getActiveMinProcId</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.348">getActiveMinProcId</a>()</pre>
 </li>
 </ul>
 <a name="getActiveMaxProcId--">
@@ -706,7 +706,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getActiveMaxProcId</h4>
-<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.366">getActiveMaxProcId</a>()</pre>
+<pre>public&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.368">getActiveMaxProcId</a>()</pre>
 </li>
 </ul>
 <a name="getBitmapIndex-long-">
@@ -715,7 +715,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getBitmapIndex</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.388">getBitmapIndex</a>(long&nbsp;procId)</pre>
+<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.390">getBitmapIndex</a>(long&nbsp;procId)</pre>
 </li>
 </ul>
 <a name="updateState-long-boolean-">
@@ -724,7 +724,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>updateState</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.392">updateState</a>(long&nbsp;procId,
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.394">updateState</a>(long&nbsp;procId,
                  boolean&nbsp;isDeleted)</pre>
 </li>
 </ul>
@@ -734,7 +734,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>alignUp</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.411">alignUp</a>(long&nbsp;x)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.413">alignUp</a>(long&nbsp;x)</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>upper boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.</dd>
@@ -747,7 +747,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>alignDown</h4>
-<pre>private static&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.418">alignDown</a>(long&nbsp;x)</pre>
+<pre>private static&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html#line.420">alignDown</a>(long&nbsp;x)</pre>
 <dl>
 <dt><span class="returnLabel">Returns:</span></dt>
 <dd>lower boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.</dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
index 7833205..11cd43a 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>public static enum <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.63">ProcedureStoreTracker.DeleteState</a>
+<pre>public static enum <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html#line.64">ProcedureStoreTracker.DeleteState</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang">Enum</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>&gt;</pre>
 </li>
 </ul>
@@ -213,7 +213,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>YES</h4>
-<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.63">YES</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.64">YES</a></pre>
 </li>
 </ul>
 <a name="NO">
@@ -222,7 +222,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>NO</h4>
-<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.63">NO</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.64">NO</a></pre>
 </li>
 </ul>
 <a name="MAYBE">
@@ -231,7 +231,7 @@ the order they are declared.</div>
 <ul class="blockListLast">
 <li class="blockList">
 <h4>MAYBE</h4>
-<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.63">MAYBE</a></pre>
+<pre>public static final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.64">MAYBE</a></pre>
 </li>
 </ul>
 </li>
@@ -248,7 +248,7 @@ the order they are declared.</div>
 <ul class="blockList">
 <li class="blockList">
 <h4>values</h4>
-<pre>public static&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.63">values</a>()</pre>
+<pre>public static&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.64">values</a>()</pre>
 <div class="block">Returns an array containing the constants of this enum type, in
 the order they are declared.  This method may be used to iterate
 over the constants as follows:
@@ -268,7 +268,7 @@ for (ProcedureStoreTracker.DeleteState c : ProcedureStoreTracker.DeleteState.val
 <ul class="blockListLast">
 <li class="blockList">
 <h4>valueOf</h4>
-<pre>public static&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.63">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
+<pre>public static&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html" title="enum in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker.DeleteState</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html#line.64">valueOf</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;name)</pre>
 <div class="block">Returns the enum constant of this type with the specified name.
 The string must match <i>exactly</i> an identifier used to declare an
 enum constant in this type.  (Extraneous whitespace characters are 


[30/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a

<TRUNCATED>

[47/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
index ff0d261..d158693 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1168">StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1176">StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Compute the cost of a potential cluster state from skew in number of
  primary regions on a cluster.</div>
@@ -230,7 +230,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>PRIMARY_REGION_COUNT_SKEW_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1169">PRIMARY_REGION_COUNT_SKEW_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1177">PRIMARY_REGION_COUNT_SKEW_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.PRIMARY_REGION_COUNT_SKEW_COST_KEY">Constant Field Values</a></dd>
@@ -243,7 +243,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_PRIMARY_REGION_COUNT_SKEW_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1171">DEFAULT_PRIMARY_REGION_COUNT_SKEW_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1179">DEFAULT_PRIMARY_REGION_COUNT_SKEW_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.DEFAULT_PRIMARY_REGION_COUNT_SKEW_COST">Constant Field Values</a></dd>
@@ -256,7 +256,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>stats</h4>
-<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1173">stats</a></pre>
+<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1181">stats</a></pre>
 </li>
 </ul>
 </li>
@@ -273,7 +273,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>PrimaryRegionCountSkewCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1175">PrimaryRegionCountSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1183">PrimaryRegionCountSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -290,7 +290,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1183">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html#line.1191">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
index 1c5ed65..033f804 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1338">StochasticLoadBalancer.RackLocalityCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1346">StochasticLoadBalancer.RackLocalityCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.LocalityBasedCostFunction</a></pre>
 </li>
 </ul>
@@ -239,7 +239,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>RACK_LOCALITY_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1340">RACK_LOCALITY_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1348">RACK_LOCALITY_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RackLocalityCostFunction.RACK_LOCALITY_COST_KEY">Constant Field Values</a></dd>
@@ -252,7 +252,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_RACK_LOCALITY_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1341">DEFAULT_RACK_LOCALITY_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1349">DEFAULT_RACK_LOCALITY_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RackLocalityCostFunction.DEFAULT_RACK_LOCALITY_COST">Constant Field Values</a></dd>
@@ -273,7 +273,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RackLocalityCostFunction</h4>
-<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1343">RackLocalityCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1351">RackLocalityCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                                 <a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;services)</pre>
 </li>
 </ul>
@@ -291,7 +291,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>regionIndexToEntityIndex</h4>
-<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1354">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
+<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html#line.1362">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#regionIndexToEntityIndex-int-">StochasticLoadBalancer.LocalityBasedCostFunction</a></code></span></div>
 <div class="block">Maps region to the current entity (server or rack) on which it is stored</div>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
index 83ec0f5..797c5a6 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.721">StochasticLoadBalancer.RandomCandidateGenerator</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.729">StochasticLoadBalancer.RandomCandidateGenerator</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></pre>
 </li>
 </ul>
@@ -194,7 +194,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RandomCandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html#line.721">RandomCandidateGenerator</a>()</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html#line.729">RandomCandidateGenerator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -211,7 +211,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>generate</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html#line.724">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html#line.732">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#generate-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">generate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
index 3028c9b..509725e 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
@@ -128,7 +128,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1457">StochasticLoadBalancer.ReadRequestCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1465">StochasticLoadBalancer.ReadRequestCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a></pre>
 <div class="block">Compute the cost of total number of read requests  The more unbalanced the higher the
  computed cost will be.  This uses a rolling average of regionload.</div>
@@ -250,7 +250,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>READ_REQUEST_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1459">READ_REQUEST_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1467">READ_REQUEST_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.ReadRequestCostFunction.READ_REQUEST_COST_KEY">Constant Field Values</a></dd>
@@ -263,7 +263,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_READ_REQUEST_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1461">DEFAULT_READ_REQUEST_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1469">DEFAULT_READ_REQUEST_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.ReadRequestCostFunction.DEFAULT_READ_REQUEST_COST">Constant Field Values</a></dd>
@@ -284,7 +284,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>ReadRequestCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1463">ReadRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1471">ReadRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -301,7 +301,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1469">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html#line.1477">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">getCostFromRl</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
index 9df4fbd..826bbae 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1137">StochasticLoadBalancer.RegionCountSkewCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1145">StochasticLoadBalancer.RegionCountSkewCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Compute the cost of a potential cluster state from skew in number of
  regions on a cluster.</div>
@@ -230,7 +230,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>REGION_COUNT_SKEW_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1138">REGION_COUNT_SKEW_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1146">REGION_COUNT_SKEW_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionCountSkewCostFunction.REGION_COUNT_SKEW_COST_KEY">Constant Field Values</a></dd>
@@ -243,7 +243,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_REGION_COUNT_SKEW_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1140">DEFAULT_REGION_COUNT_SKEW_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1148">DEFAULT_REGION_COUNT_SKEW_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionCountSkewCostFunction.DEFAULT_REGION_COUNT_SKEW_COST">Constant Field Values</a></dd>
@@ -256,7 +256,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>stats</h4>
-<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1142">stats</a></pre>
+<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1150">stats</a></pre>
 </li>
 </ul>
 </li>
@@ -273,7 +273,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RegionCountSkewCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1144">RegionCountSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1152">RegionCountSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -290,7 +290,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1151">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionCountSkewCostFunction.html#line.1159">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
index fd63f54..61da737 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.850">StochasticLoadBalancer.RegionReplicaCandidateGenerator</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.858">StochasticLoadBalancer.RegionReplicaCandidateGenerator</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></pre>
 <div class="block">Generates candidates which moves the replicas out of the region server for
  co-hosted region replicas</div>
@@ -228,7 +228,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>randomGenerator</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.RandomCandidateGenerator</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.852">randomGenerator</a></pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RandomCandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.RandomCandidateGenerator</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.860">randomGenerator</a></pre>
 </li>
 </ul>
 </li>
@@ -245,7 +245,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RegionReplicaCandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.850">RegionReplicaCandidateGenerator</a>()</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.858">RegionReplicaCandidateGenerator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -262,7 +262,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>selectCoHostedRegionPerGroup</h4>
-<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.863">selectCoHostedRegionPerGroup</a>(int[]&nbsp;primariesOfRegionsPerGroup,
+<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.871">selectCoHostedRegionPerGroup</a>(int[]&nbsp;primariesOfRegionsPerGroup,
                                  int[]&nbsp;regionsPerGroup,
                                  int[]&nbsp;regionIndexToPrimaryIndex)</pre>
 <div class="block">Randomly select one regionIndex out of all region replicas co-hosted in the same group
@@ -284,7 +284,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>generate</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.907">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#line.915">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#generate-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">generate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
index 76e8473..5700a2e 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1523">StochasticLoadBalancer.RegionReplicaHostCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1531">StochasticLoadBalancer.RegionReplicaHostCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">A cost function for region replicas. We give a very high cost to hosting
  replicas of the same region in the same host. We do not prevent the case
@@ -272,7 +272,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>REGION_REPLICA_HOST_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1524">REGION_REPLICA_HOST_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1532">REGION_REPLICA_HOST_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionReplicaHostCostFunction.REGION_REPLICA_HOST_COST_KEY">Constant Field Values</a></dd>
@@ -285,7 +285,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_REGION_REPLICA_HOST_COST_KEY</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1526">DEFAULT_REGION_REPLICA_HOST_COST_KEY</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1534">DEFAULT_REGION_REPLICA_HOST_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionReplicaHostCostFunction.DEFAULT_REGION_REPLICA_HOST_COST_KEY">Constant Field Values</a></dd>
@@ -298,7 +298,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>maxCost</h4>
-<pre>long <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1528">maxCost</a></pre>
+<pre>long <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1536">maxCost</a></pre>
 </li>
 </ul>
 <a name="costsPerGroup">
@@ -307,7 +307,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>costsPerGroup</h4>
-<pre>long[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1529">costsPerGroup</a></pre>
+<pre>long[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1537">costsPerGroup</a></pre>
 </li>
 </ul>
 <a name="primariesOfRegionsPerGroup">
@@ -316,7 +316,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>primariesOfRegionsPerGroup</h4>
-<pre>int[][] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1530">primariesOfRegionsPerGroup</a></pre>
+<pre>int[][] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1538">primariesOfRegionsPerGroup</a></pre>
 </li>
 </ul>
 </li>
@@ -333,7 +333,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RegionReplicaHostCostFunction</h4>
-<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1532">RegionReplicaHostCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1540">RegionReplicaHostCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -350,7 +350,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>init</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1539">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1547">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#init-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">StochasticLoadBalancer.CostFunction</a></code></span></div>
 <div class="block">Called once per LB invocation to give the cost function
  to initialize it's state, and perform any costly calculation.</div>
@@ -366,7 +366,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>getMaxCost</h4>
-<pre>long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1552">getMaxCost</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1560">getMaxCost</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 </li>
 </ul>
 <a name="isNeeded--">
@@ -375,7 +375,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>isNeeded</h4>
-<pre>boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1568">isNeeded</a>()</pre>
+<pre>boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1576">isNeeded</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#isNeeded--">isNeeded</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>
@@ -388,7 +388,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1573">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1581">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>
@@ -401,7 +401,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>costPerGroup</h4>
-<pre>protected&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1593">costPerGroup</a>(int[]&nbsp;primariesOfRegions)</pre>
+<pre>protected&nbsp;long&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1601">costPerGroup</a>(int[]&nbsp;primariesOfRegions)</pre>
 <div class="block">For each primary region, it computes the total number of replicas in the array (numReplicas)
  and returns a sum of numReplicas-1 squared. For example, if the server hosts
  regions a, b, c, d, e, f where a and b are same replicas, and c,d,e are same replicas, it
@@ -420,7 +420,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>regionMoved</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1616">regionMoved</a>(int&nbsp;region,
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html#line.1624">regionMoved</a>(int&nbsp;region,
                            int&nbsp;oldServer,
                            int&nbsp;newServer)</pre>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
index 27d56b0..0c2c6d4 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.934">StochasticLoadBalancer.RegionReplicaRackCandidateGenerator</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.942">StochasticLoadBalancer.RegionReplicaRackCandidateGenerator</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.RegionReplicaCandidateGenerator</a></pre>
 <div class="block">Generates candidates which moves the replicas out of the rack for
  co-hosted region replicas in the same rack</div>
@@ -223,7 +223,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RegionReplicaRackCandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html#line.934">RegionReplicaRackCandidateGenerator</a>()</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html#line.942">RegionReplicaRackCandidateGenerator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -240,7 +240,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>generate</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html#line.936">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCandidateGenerator.html#line.944">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html#generate-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">generate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.RegionReplicaCandidateGenerator</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
index fe6f10d..970e264 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1639">StochasticLoadBalancer.RegionReplicaRackCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1647">StochasticLoadBalancer.RegionReplicaRackCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaHostCostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.RegionReplicaHostCostFunction</a></pre>
 <div class="block">A cost function for region replicas for the rack distribution. We give a relatively high
  cost to hosting replicas of the same region in the same rack. We do not prevent the case
@@ -255,7 +255,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>REGION_REPLICA_RACK_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1640">REGION_REPLICA_RACK_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1648">REGION_REPLICA_RACK_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionReplicaRackCostFunction.REGION_REPLICA_RACK_COST_KEY">Constant Field Values</a></dd>
@@ -268,7 +268,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_REGION_REPLICA_RACK_COST_KEY</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1642">DEFAULT_REGION_REPLICA_RACK_COST_KEY</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1650">DEFAULT_REGION_REPLICA_RACK_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.RegionReplicaRackCostFunction.DEFAULT_REGION_REPLICA_RACK_COST_KEY">Constant Field Values</a></dd>
@@ -289,7 +289,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>RegionReplicaRackCostFunction</h4>
-<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1644">RegionReplicaRackCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre>public&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1652">RegionReplicaRackCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -306,7 +306,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>init</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1651">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1659">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#init-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">StochasticLoadBalancer.CostFunction</a></code></span></div>
 <div class="block">Called once per LB invocation to give the cost function
  to initialize it's state, and perform any costly calculation.</div>
@@ -322,7 +322,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>regionMoved</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1666">regionMoved</a>(int&nbsp;region,
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html#line.1674">regionMoved</a>(int&nbsp;region,
                            int&nbsp;oldServer,
                            int&nbsp;newServer)</pre>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
index 4978909..d01ff53 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1317">StochasticLoadBalancer.ServerLocalityCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1325">StochasticLoadBalancer.ServerLocalityCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.LocalityBasedCostFunction</a></pre>
 </li>
 </ul>
@@ -239,7 +239,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>LOCALITY_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1319">LOCALITY_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1327">LOCALITY_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.ServerLocalityCostFunction.LOCALITY_COST_KEY">Constant Field Values</a></dd>
@@ -252,7 +252,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_LOCALITY_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1320">DEFAULT_LOCALITY_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1328">DEFAULT_LOCALITY_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.ServerLocalityCostFunction.DEFAULT_LOCALITY_COST">Constant Field Values</a></dd>
@@ -273,7 +273,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>ServerLocalityCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1322">ServerLocalityCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1330">ServerLocalityCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                            <a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;srv)</pre>
 </li>
 </ul>
@@ -291,7 +291,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>regionIndexToEntityIndex</h4>
-<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1333">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
+<pre>int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html#line.1341">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#regionIndexToEntityIndex-int-">StochasticLoadBalancer.LocalityBasedCostFunction</a></code></span></div>
 <div class="block">Maps region to the current entity (server or rack) on which it is stored</div>
 <dl>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
index ec51863..477d4a8 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1704">StochasticLoadBalancer.StoreFileCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1712">StochasticLoadBalancer.StoreFileCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></pre>
 <div class="block">Compute the cost of total open storefiles size.  The more unbalanced the higher the
  computed cost will be.  This uses a rolling average of regionload.</div>
@@ -238,7 +238,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>STOREFILE_SIZE_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1706">STOREFILE_SIZE_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1714">STOREFILE_SIZE_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.StoreFileCostFunction.STOREFILE_SIZE_COST_KEY">Constant Field Values</a></dd>
@@ -251,7 +251,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_STOREFILE_SIZE_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1708">DEFAULT_STOREFILE_SIZE_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1716">DEFAULT_STOREFILE_SIZE_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.StoreFileCostFunction.DEFAULT_STOREFILE_SIZE_COST">Constant Field Values</a></dd>
@@ -272,7 +272,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>StoreFileCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1710">StoreFileCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1718">StoreFileCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -289,7 +289,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1716">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html#line.1724">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">getCostFromRl</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
index de1421f..3fca414 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1208">StochasticLoadBalancer.TableSkewCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1216">StochasticLoadBalancer.TableSkewCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Compute the cost of a potential cluster configuration based upon how evenly
  distributed tables are.</div>
@@ -226,7 +226,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>TABLE_SKEW_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1210">TABLE_SKEW_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1218">TABLE_SKEW_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.TableSkewCostFunction.TABLE_SKEW_COST_KEY">Constant Field Values</a></dd>
@@ -239,7 +239,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_TABLE_SKEW_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1212">DEFAULT_TABLE_SKEW_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1220">DEFAULT_TABLE_SKEW_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.TableSkewCostFunction.DEFAULT_TABLE_SKEW_COST">Constant Field Values</a></dd>
@@ -260,7 +260,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>TableSkewCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1214">TableSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1222">TableSkewCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -277,7 +277,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1220">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.TableSkewCostFunction.html#line.1228">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
index e533225..aba0b45 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html
@@ -128,7 +128,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1500">StochasticLoadBalancer.WriteRequestCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1508">StochasticLoadBalancer.WriteRequestCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a></pre>
 <div class="block">Compute the cost of total number of write requests.  The more unbalanced the higher the
  computed cost will be.  This uses a rolling average of regionload.</div>
@@ -250,7 +250,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>WRITE_REQUEST_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1502">WRITE_REQUEST_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1510">WRITE_REQUEST_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.WriteRequestCostFunction.WRITE_REQUEST_COST_KEY">Constant Field Values</a></dd>
@@ -263,7 +263,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_WRITE_REQUEST_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1504">DEFAULT_WRITE_REQUEST_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1512">DEFAULT_WRITE_REQUEST_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.WriteRequestCostFunction.DEFAULT_WRITE_REQUEST_COST">Constant Field Values</a></dd>
@@ -284,7 +284,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>WriteRequestCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1506">WriteRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1514">WriteRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -301,7 +301,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1512">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.WriteRequestCostFunction.html#line.1520">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">getCostFromRl</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
index c6f653e..4d8d1ce 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
@@ -1162,7 +1162,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>updateStochasticCosts</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.452">updateStochasticCosts</a>(<a href="../../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.460">updateStochasticCosts</a>(<a href="../../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName,
                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Double.html?is-external=true" title="class or interface in java.lang">Double</a>&nbsp;overall,
                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Double.html?is-external=true" title="class or interface in java.lang">Double</a>[]&nbsp;subCosts)</pre>
 <div class="block">update costs to JMX</div>
@@ -1174,7 +1174,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>functionCost</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.474">functionCost</a>()</pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.482">functionCost</a>()</pre>
 </li>
 </ul>
 <a name="createRegionPlans-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">
@@ -1183,7 +1183,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>createRegionPlans</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/RegionPlan.html" title="class in org.apache.hadoop.hbase.master">RegionPlan</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.494">createRegionPlans</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/RegionPlan.html" title="class in org.apache.hadoop.hbase.master">RegionPlan</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.502">createRegionPlans</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <div class="block">Create all of the RegionPlan's needed to move from the initial cluster state to the desired
  state.</div>
 <dl>
@@ -1200,7 +1200,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>updateRegionLoad</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.520">updateRegionLoad</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.528">updateRegionLoad</a>()</pre>
 <div class="block">Store the current region loads.</div>
 </li>
 </ul>
@@ -1210,7 +1210,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>initCosts</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.544">initCosts</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.552">initCosts</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 </li>
 </ul>
 <a name="updateCostsWithAction-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action-">
@@ -1219,7 +1219,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>updateCostsWithAction</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.550">updateCostsWithAction</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.558">updateCostsWithAction</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                      <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;action)</pre>
 </li>
 </ul>
@@ -1229,7 +1229,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>getCostFunctionNames</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.559">getCostFunctionNames</a>()</pre>
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>[]&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.567">getCostFunctionNames</a>()</pre>
 <div class="block">Get the names of the cost functions</div>
 </li>
 </ul>
@@ -1239,7 +1239,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockList">
 <li class="blockList">
 <h4>computeCost</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.579">computeCost</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.587">computeCost</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                              double&nbsp;previousCost)</pre>
 <div class="block">This is the main cost function.  It will compute a cost associated with a proposed cluster
  state.  All different costs will be combined with their multipliers to produce a double cost.</div>
@@ -1259,7 +1259,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseL
 <ul class="blockListLast">
 <li class="blockList">
 <h4>composeAttributeName</h4>
-<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1724">composeAttributeName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;tableName,
+<pre>public static&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1732">composeAttributeName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;tableName,
                                           <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;costFunctionName)</pre>
 <div class="block">A helper function to compose the attribute name from tablename and costfunction name</div>
 </li>


[25/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RackLocalityCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class=

<TRUNCATED>

[06/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
index 9b964f6..98ef11a 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.PushType.html
@@ -105,7 +105,7 @@
 <span class="sourceLineNo">097</span> * will first be initialized to the oldest file's tracker(which is stored in the trailer), using the<a name="line.97"></a>
 <span class="sourceLineNo">098</span> * method {@link ProcedureStoreTracker#resetTo(ProcedureStoreTracker, boolean)}, and then merge it<a name="line.98"></a>
 <span class="sourceLineNo">099</span> * with the tracker of every newer wal files, using the<a name="line.99"></a>
-<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)}.<a name="line.100"></a>
+<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker)}.<a name="line.100"></a>
 <span class="sourceLineNo">101</span> * If we find out<a name="line.101"></a>
 <span class="sourceLineNo">102</span> * that all the modified procedures for the oldest wal file are modified or deleted in newer wal<a name="line.102"></a>
 <span class="sourceLineNo">103</span> * files, then we can delete it. This is because that, every time we call<a name="line.103"></a>
@@ -1181,244 +1181,243 @@
 <span class="sourceLineNo">1173</span>    }<a name="line.1173"></a>
 <span class="sourceLineNo">1174</span><a name="line.1174"></a>
 <span class="sourceLineNo">1175</span>    // compute the holding tracker.<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>    //  - the first WAL is used for the 'updates'<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span>    //  - the global tracker is passed in first to decide which procedures are not<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>    //    exist anymore, so we can mark them as deleted in holdingCleanupTracker.<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>    //    Only global tracker have the whole picture here.<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>    //  - the other WALs are scanned to remove procs already updated in a newer wal.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>    //    If it is updated in a newer wal, we can mark it as delelted in holdingCleanupTracker<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span>    //    But, we can not delete it if it was shown deleted in the newer wal, as said<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>    //    above.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span>    // TODO: exit early if holdingCleanupTracker.isEmpty()<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>    //Passing in the global tracker, we can delete the procedures not in the global<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>    //tracker, because they are deleted in the later logs<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    holdingCleanupTracker.setDeletedIfModifiedInBoth(storeTracker, true);<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    for (int i = 1, size = logs.size() - 1; i &lt; size; ++i) {<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>      // Set deleteIfNotExists to false since a single log's tracker is passed in.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>      // Since a specific procedure may not show up in the log at all(not executed or<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span>      // updated during the time), we can not delete the procedure just because this log<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>      // don't have the info of the procedure. We can delete the procedure only if<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>      // in this log's tracker, it was cleanly showed that the procedure is modified or deleted<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>      // in the corresponding BitSetNode.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(logs.get(i).getTracker(), false);<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span>    }<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>  }<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span><a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>  /**<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>   */<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    if (logs.size() &lt;= 1) {<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>      return;<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    }<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span><a name="line.1207"></a>
-<span class="sourceLineNo">1208</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span><a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>    boolean removed = false;<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    while (logs.size() &gt; 1) {<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>        break;<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>      }<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>      removeLogFile(log, walArchiveDir);<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>      removed = true;<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    }<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span><a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>    if (removed) {<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>      buildHoldingCleanupTracker();<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>    }<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>  }<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span><a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>    try {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span>      LOG.trace("Removing log={}", log);<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>      log.removeFile(walArchiveDir);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>      logs.remove(log);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>    } catch (IOException e) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>      return false;<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    }<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>    return true;<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>  }<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span><a name="line.1238"></a>
-<span class="sourceLineNo">1239</span>  // ==========================================================================<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  //  FileSystem Log Files helpers<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>  // ==========================================================================<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>  public Path getWALDir() {<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>    return this.walDir;<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span>  }<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span><a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>  @VisibleForTesting<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span>  Path getWalArchiveDir() {<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    return this.walArchiveDir;<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>  }<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span><a name="line.1250"></a>
-<span class="sourceLineNo">1251</span>  public FileSystem getFileSystem() {<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    return this.fs;<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>  }<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span><a name="line.1254"></a>
-<span class="sourceLineNo">1255</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  private static long getLogIdFromName(final String name) {<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>    int end = name.lastIndexOf(".log");<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>    return Long.parseLong(name.substring(start, end));<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span>  }<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span><a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    @Override<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>    public boolean accept(Path path) {<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span>      String name = path.getName();<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>    }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>  };<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span><a name="line.1272"></a>
-<span class="sourceLineNo">1273</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>    @Override<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>      return Long.compare(aId, bId);<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>    }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>  };<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span><a name="line.1282"></a>
-<span class="sourceLineNo">1283</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>    try {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>      return files;<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>    } catch (FileNotFoundException e) {<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>      return null;<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>    }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>  }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span><a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>  /**<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>   * the file set by log id.<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   * @return Max-LogID of the specified log file set<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>   */<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>      return 0L;<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>    }<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>  }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span><a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>  /**<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   * the file set by log id.<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   * @return Max-LogID of the specified log file set<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   */<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>      return 0L;<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>    }<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>    long maxLogId = 0;<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      final Path logPath = logFiles[i].getPath();<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>      if (!isRunning()) {<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>        throw new IOException("wal aborting");<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span>      }<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span><a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>      if (log != null) {<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>        this.logs.add(log);<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>      }<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    }<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span>    initTrackerFromOldLogs();<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    return maxLogId;<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>  }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span>  /**<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>   */<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>  private void initTrackerFromOldLogs() {<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span>      return;<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>    ProcedureWALFile log = logs.getLast();<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (!log.getTracker().isPartial()) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      storeTracker.resetTo(log.getTracker());<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>    } else {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>      storeTracker.reset();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      storeTracker.setPartialFlag(true);<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>    }<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>  }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span><a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>  /**<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>   * Loads given log file and it's tracker.<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>   */<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>      throws IOException {<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span>    if (logFile.getLen() == 0) {<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span>      log.removeFile(walArchiveDir);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>      return null;<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>    }<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span>    try {<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>      log.open();<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>      log.removeFile(walArchiveDir);<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      return null;<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span>    } catch (IOException e) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      LOG.error(msg, e);<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span>      throw new IOException(msg, e);<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span>    }<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span><a name="line.1373"></a>
-<span class="sourceLineNo">1374</span>    try {<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>      log.readTracker();<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>    } catch (IOException e) {<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>      log.getTracker().reset();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>      log.getTracker().setPartialFlag(true);<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span>    }<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span><a name="line.1381"></a>
-<span class="sourceLineNo">1382</span>    log.close();<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span>    return log;<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span>  }<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span><a name="line.1385"></a>
-<span class="sourceLineNo">1386</span>  /**<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span>   * For testing parse and profiling.<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span>   */<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span>  public static void main(String [] args) throws IOException {<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span>    if (args == null || args.length != 1) {<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span>      System.exit(-1);<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span>    }<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span>        @Override<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span>          // no-op<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span>        }<a name="line.1403"></a>
-<span class="sourceLineNo">1404</span>      });<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span>    try {<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span>      store.start(16);<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span>      pe.init(1, true);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span>    } finally {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span>      store.stop(true);<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span>    }<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span>  }<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span>}<a name="line.1413"></a>
+<span class="sourceLineNo">1176</span>    // - the first WAL is used for the 'updates'<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    // - the global tracker will be used to determine whether a procedure has been deleted<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    // - other trackers will be used to determine whether a procedure has been updated, as a deleted<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>    // procedure can always be detected by checking the global tracker, we can save the deleted<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span>    // checks when applying other trackers<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span>    holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span>    // the logs is a linked list, so avoid calling get(index) on it.<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span>    Iterator&lt;ProcedureWALFile&gt; iter = logs.iterator();<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span>    // skip the tracker for the first file when creating the iterator.<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>    iter.next();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>    ProcedureStoreTracker tracker = iter.next().getTracker();<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>    // testing iter.hasNext after calling iter.next to skip applying the tracker for last file,<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>    // which is just the storeTracker above.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>    while (iter.hasNext()) {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker);<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>      if (holdingCleanupTracker.isEmpty()) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>        break;<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>      }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span>      iter.next();<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>    }<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span>  }<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span><a name="line.1198"></a>
+<span class="sourceLineNo">1199</span>  /**<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   */<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>    if (logs.size() &lt;= 1) {<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>      return;<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>    }<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span><a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span><a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    boolean removed = false;<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>    while (logs.size() &gt; 1) {<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>        break;<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>      }<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>      removeLogFile(log, walArchiveDir);<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span>      removed = true;<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>    }<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span><a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>    if (removed) {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>      buildHoldingCleanupTracker();<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    }<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>  }<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span><a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>    try {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>      LOG.trace("Removing log={}", log);<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      log.removeFile(walArchiveDir);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>      logs.remove(log);<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>    } catch (IOException e) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>      return false;<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    }<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>    return true;<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span>  }<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span><a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>  // ==========================================================================<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>  //  FileSystem Log Files helpers<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>  // ==========================================================================<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  public Path getWALDir() {<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>    return this.walDir;<a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span><a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>  @VisibleForTesting<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  Path getWalArchiveDir() {<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span>    return this.walArchiveDir;<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span>  }<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span><a name="line.1249"></a>
+<span class="sourceLineNo">1250</span>  public FileSystem getFileSystem() {<a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    return this.fs;<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>  }<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span><a name="line.1253"></a>
+<span class="sourceLineNo">1254</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>  }<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span><a name="line.1257"></a>
+<span class="sourceLineNo">1258</span>  private static long getLogIdFromName(final String name) {<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span>    int end = name.lastIndexOf(".log");<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span>    return Long.parseLong(name.substring(start, end));<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span><a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>    @Override<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span>    public boolean accept(Path path) {<a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>      String name = path.getName();<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    }<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  };<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    @Override<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>      return Long.compare(aId, bId);<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    }<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  };<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>    try {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span>      return files;<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>    } catch (FileNotFoundException e) {<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>      return null;<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>    }<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>  }<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span><a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>  /**<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   * the file set by log id.<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   * @return Max-LogID of the specified log file set<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>   */<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>      return 0L;<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>    }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>  }<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span><a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>  /**<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   * the file set by log id.<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   * @return Max-LogID of the specified log file set<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   */<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>      return 0L;<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>    }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span>    long maxLogId = 0;<a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>      final Path logPath = logFiles[i].getPath();<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>      if (!isRunning()) {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>        throw new IOException("wal aborting");<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      }<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span><a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span>      if (log != null) {<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>        this.logs.add(log);<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>      }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    }<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    initTrackerFromOldLogs();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    return maxLogId;<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>  }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span><a name="line.1331"></a>
+<span class="sourceLineNo">1332</span>  /**<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>   */<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span>  private void initTrackerFromOldLogs() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>      return;<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>    }<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    ProcedureWALFile log = logs.getLast();<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>    if (!log.getTracker().isPartial()) {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span>      storeTracker.resetTo(log.getTracker());<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>    } else {<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>      storeTracker.reset();<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>      storeTracker.setPartialFlag(true);<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>    }<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>  }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>  /**<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>   * Loads given log file and it's tracker.<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>   */<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>      throws IOException {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>    if (logFile.getLen() == 0) {<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>      log.removeFile(walArchiveDir);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>      return null;<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span>    }<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span>    try {<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>      log.open();<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span>      log.removeFile(walArchiveDir);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>      return null;<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>    } catch (IOException e) {<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span>      LOG.error(msg, e);<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      throw new IOException(msg, e);<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span><a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>    try {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>      log.readTracker();<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span>    } catch (IOException e) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>      log.getTracker().reset();<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span>      log.getTracker().setPartialFlag(true);<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span>    }<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span><a name="line.1380"></a>
+<span class="sourceLineNo">1381</span>    log.close();<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span>    return log;<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span>  }<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span><a name="line.1384"></a>
+<span class="sourceLineNo">1385</span>  /**<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span>   * For testing parse and profiling.<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span>   */<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span>  public static void main(String [] args) throws IOException {<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span>    if (args == null || args.length != 1) {<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span>      System.exit(-1);<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span>    }<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span>        @Override<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span>          // no-op<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span>        }<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span>      });<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span>    try {<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span>      store.start(16);<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span>      pe.init(1, true);<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span>    } finally {<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span>      store.stop(true);<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span>    }<a name="line.1410"></a>
+<span class="sourceLineNo">1411</span>  }<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span>}<a name="line.1412"></a>
 
 
 


[02/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
index 3ec2d53..9400177 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
@@ -33,390 +33,433 @@
 <span class="sourceLineNo">025</span>import java.util.HashMap;<a name="line.25"></a>
 <span class="sourceLineNo">026</span>import java.util.List;<a name="line.26"></a>
 <span class="sourceLineNo">027</span>import java.util.Map;<a name="line.27"></a>
-<span class="sourceLineNo">028</span><a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.hadoop.conf.Configuration;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import org.apache.hadoop.fs.FileStatus;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.apache.hadoop.fs.FileSystem;<a name="line.31"></a>
-<span class="sourceLineNo">032</span>import org.apache.hadoop.fs.FileUtil;<a name="line.32"></a>
-<span class="sourceLineNo">033</span>import org.apache.hadoop.fs.Path;<a name="line.33"></a>
-<span class="sourceLineNo">034</span>import org.apache.hadoop.fs.permission.FsPermission;<a name="line.34"></a>
-<span class="sourceLineNo">035</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.35"></a>
-<span class="sourceLineNo">036</span>import org.apache.hadoop.hbase.HConstants;<a name="line.36"></a>
-<span class="sourceLineNo">037</span>import org.apache.hadoop.hbase.TableName;<a name="line.37"></a>
-<span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.38"></a>
-<span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.ipc.RpcServer;<a name="line.39"></a>
-<span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;<a name="line.40"></a>
-<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.security.User;<a name="line.41"></a>
-<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.42"></a>
-<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.security.token.FsDelegationToken;<a name="line.43"></a>
-<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.security.token.TokenUtil;<a name="line.44"></a>
-<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.45"></a>
-<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.util.FSHDFSUtils;<a name="line.46"></a>
-<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.47"></a>
-<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.util.Methods;<a name="line.48"></a>
-<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.49"></a>
-<span class="sourceLineNo">050</span>import org.apache.hadoop.io.Text;<a name="line.50"></a>
-<span class="sourceLineNo">051</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>import org.apache.hadoop.security.token.Token;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>import org.slf4j.Logger;<a name="line.54"></a>
-<span class="sourceLineNo">055</span>import org.slf4j.LoggerFactory;<a name="line.55"></a>
-<span class="sourceLineNo">056</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;<a name="line.56"></a>
-<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;<a name="line.57"></a>
-<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;<a name="line.58"></a>
-<span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;<a name="line.59"></a>
-<span class="sourceLineNo">060</span><a name="line.60"></a>
-<span class="sourceLineNo">061</span>/**<a name="line.61"></a>
-<span class="sourceLineNo">062</span> * Bulk loads in secure mode.<a name="line.62"></a>
-<span class="sourceLineNo">063</span> *<a name="line.63"></a>
-<span class="sourceLineNo">064</span> * This service addresses two issues:<a name="line.64"></a>
-<span class="sourceLineNo">065</span> * &lt;ol&gt;<a name="line.65"></a>
-<span class="sourceLineNo">066</span> * &lt;li&gt;Moving files in a secure filesystem wherein the HBase Client<a name="line.66"></a>
-<span class="sourceLineNo">067</span> * and HBase Server are different filesystem users.&lt;/li&gt;<a name="line.67"></a>
-<span class="sourceLineNo">068</span> * &lt;li&gt;Does moving in a secure manner. Assuming that the filesystem<a name="line.68"></a>
-<span class="sourceLineNo">069</span> * is POSIX compliant.&lt;/li&gt;<a name="line.69"></a>
-<span class="sourceLineNo">070</span> * &lt;/ol&gt;<a name="line.70"></a>
-<span class="sourceLineNo">071</span> *<a name="line.71"></a>
-<span class="sourceLineNo">072</span> * The algorithm is as follows:<a name="line.72"></a>
-<span class="sourceLineNo">073</span> * &lt;ol&gt;<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * &lt;li&gt;Create an hbase owned staging directory which is<a name="line.74"></a>
-<span class="sourceLineNo">075</span> * world traversable (711): {@code /hbase/staging}&lt;/li&gt;<a name="line.75"></a>
-<span class="sourceLineNo">076</span> * &lt;li&gt;A user writes out data to his secure output directory: {@code /user/foo/data}&lt;/li&gt;<a name="line.76"></a>
-<span class="sourceLineNo">077</span> * &lt;li&gt;A call is made to hbase to create a secret staging directory<a name="line.77"></a>
-<span class="sourceLineNo">078</span> * which globally rwx (777): {@code /user/staging/averylongandrandomdirectoryname}&lt;/li&gt;<a name="line.78"></a>
-<span class="sourceLineNo">079</span> * &lt;li&gt;The user moves the data into the random staging directory,<a name="line.79"></a>
-<span class="sourceLineNo">080</span> * then calls bulkLoadHFiles()&lt;/li&gt;<a name="line.80"></a>
-<span class="sourceLineNo">081</span> * &lt;/ol&gt;<a name="line.81"></a>
-<span class="sourceLineNo">082</span> *<a name="line.82"></a>
-<span class="sourceLineNo">083</span> * Like delegation tokens the strength of the security lies in the length<a name="line.83"></a>
-<span class="sourceLineNo">084</span> * and randomness of the secret directory.<a name="line.84"></a>
-<span class="sourceLineNo">085</span> *<a name="line.85"></a>
-<span class="sourceLineNo">086</span> */<a name="line.86"></a>
-<span class="sourceLineNo">087</span>@InterfaceAudience.Private<a name="line.87"></a>
-<span class="sourceLineNo">088</span>public class SecureBulkLoadManager {<a name="line.88"></a>
-<span class="sourceLineNo">089</span><a name="line.89"></a>
-<span class="sourceLineNo">090</span>  public static final long VERSION = 0L;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span>  //320/5 = 64 characters<a name="line.92"></a>
-<span class="sourceLineNo">093</span>  private static final int RANDOM_WIDTH = 320;<a name="line.93"></a>
-<span class="sourceLineNo">094</span>  private static final int RANDOM_RADIX = 32;<a name="line.94"></a>
+<span class="sourceLineNo">028</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import java.util.function.BiFunction;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import java.util.function.Consumer;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.hadoop.conf.Configuration;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import org.apache.hadoop.fs.FileStatus;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import org.apache.hadoop.fs.FileSystem;<a name="line.34"></a>
+<span class="sourceLineNo">035</span>import org.apache.hadoop.fs.FileUtil;<a name="line.35"></a>
+<span class="sourceLineNo">036</span>import org.apache.hadoop.fs.Path;<a name="line.36"></a>
+<span class="sourceLineNo">037</span>import org.apache.hadoop.fs.permission.FsPermission;<a name="line.37"></a>
+<span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.38"></a>
+<span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.HConstants;<a name="line.39"></a>
+<span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.TableName;<a name="line.40"></a>
+<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.41"></a>
+<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.ipc.RpcServer;<a name="line.42"></a>
+<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;<a name="line.43"></a>
+<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.security.User;<a name="line.44"></a>
+<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.45"></a>
+<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.security.token.FsDelegationToken;<a name="line.46"></a>
+<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.security.token.TokenUtil;<a name="line.47"></a>
+<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.48"></a>
+<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.util.FSHDFSUtils;<a name="line.49"></a>
+<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.50"></a>
+<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.Methods;<a name="line.51"></a>
+<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.52"></a>
+<span class="sourceLineNo">053</span>import org.apache.hadoop.io.Text;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.54"></a>
+<span class="sourceLineNo">055</span>import org.apache.hadoop.security.token.Token;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>import org.slf4j.Logger;<a name="line.57"></a>
+<span class="sourceLineNo">058</span>import org.slf4j.LoggerFactory;<a name="line.58"></a>
+<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.59"></a>
+<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;<a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest;<a name="line.63"></a>
+<span class="sourceLineNo">064</span><a name="line.64"></a>
+<span class="sourceLineNo">065</span>/**<a name="line.65"></a>
+<span class="sourceLineNo">066</span> * Bulk loads in secure mode.<a name="line.66"></a>
+<span class="sourceLineNo">067</span> *<a name="line.67"></a>
+<span class="sourceLineNo">068</span> * This service addresses two issues:<a name="line.68"></a>
+<span class="sourceLineNo">069</span> * &lt;ol&gt;<a name="line.69"></a>
+<span class="sourceLineNo">070</span> * &lt;li&gt;Moving files in a secure filesystem wherein the HBase Client<a name="line.70"></a>
+<span class="sourceLineNo">071</span> * and HBase Server are different filesystem users.&lt;/li&gt;<a name="line.71"></a>
+<span class="sourceLineNo">072</span> * &lt;li&gt;Does moving in a secure manner. Assuming that the filesystem<a name="line.72"></a>
+<span class="sourceLineNo">073</span> * is POSIX compliant.&lt;/li&gt;<a name="line.73"></a>
+<span class="sourceLineNo">074</span> * &lt;/ol&gt;<a name="line.74"></a>
+<span class="sourceLineNo">075</span> *<a name="line.75"></a>
+<span class="sourceLineNo">076</span> * The algorithm is as follows:<a name="line.76"></a>
+<span class="sourceLineNo">077</span> * &lt;ol&gt;<a name="line.77"></a>
+<span class="sourceLineNo">078</span> * &lt;li&gt;Create an hbase owned staging directory which is<a name="line.78"></a>
+<span class="sourceLineNo">079</span> * world traversable (711): {@code /hbase/staging}&lt;/li&gt;<a name="line.79"></a>
+<span class="sourceLineNo">080</span> * &lt;li&gt;A user writes out data to his secure output directory: {@code /user/foo/data}&lt;/li&gt;<a name="line.80"></a>
+<span class="sourceLineNo">081</span> * &lt;li&gt;A call is made to hbase to create a secret staging directory<a name="line.81"></a>
+<span class="sourceLineNo">082</span> * which globally rwx (777): {@code /user/staging/averylongandrandomdirectoryname}&lt;/li&gt;<a name="line.82"></a>
+<span class="sourceLineNo">083</span> * &lt;li&gt;The user moves the data into the random staging directory,<a name="line.83"></a>
+<span class="sourceLineNo">084</span> * then calls bulkLoadHFiles()&lt;/li&gt;<a name="line.84"></a>
+<span class="sourceLineNo">085</span> * &lt;/ol&gt;<a name="line.85"></a>
+<span class="sourceLineNo">086</span> *<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * Like delegation tokens the strength of the security lies in the length<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * and randomness of the secret directory.<a name="line.88"></a>
+<span class="sourceLineNo">089</span> *<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span>@InterfaceAudience.Private<a name="line.91"></a>
+<span class="sourceLineNo">092</span>public class SecureBulkLoadManager {<a name="line.92"></a>
+<span class="sourceLineNo">093</span><a name="line.93"></a>
+<span class="sourceLineNo">094</span>  public static final long VERSION = 0L;<a name="line.94"></a>
 <span class="sourceLineNo">095</span><a name="line.95"></a>
-<span class="sourceLineNo">096</span>  private static final Logger LOG = LoggerFactory.getLogger(SecureBulkLoadManager.class);<a name="line.96"></a>
-<span class="sourceLineNo">097</span><a name="line.97"></a>
-<span class="sourceLineNo">098</span>  private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");<a name="line.98"></a>
-<span class="sourceLineNo">099</span>  private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");<a name="line.99"></a>
-<span class="sourceLineNo">100</span>  private SecureRandom random;<a name="line.100"></a>
-<span class="sourceLineNo">101</span>  private FileSystem fs;<a name="line.101"></a>
-<span class="sourceLineNo">102</span>  private Configuration conf;<a name="line.102"></a>
-<span class="sourceLineNo">103</span><a name="line.103"></a>
-<span class="sourceLineNo">104</span>  //two levels so it doesn't get deleted accidentally<a name="line.104"></a>
-<span class="sourceLineNo">105</span>  //no sticky bit in Hadoop 1.0<a name="line.105"></a>
-<span class="sourceLineNo">106</span>  private Path baseStagingDir;<a name="line.106"></a>
+<span class="sourceLineNo">096</span>  //320/5 = 64 characters<a name="line.96"></a>
+<span class="sourceLineNo">097</span>  private static final int RANDOM_WIDTH = 320;<a name="line.97"></a>
+<span class="sourceLineNo">098</span>  private static final int RANDOM_RADIX = 32;<a name="line.98"></a>
+<span class="sourceLineNo">099</span><a name="line.99"></a>
+<span class="sourceLineNo">100</span>  private static final Logger LOG = LoggerFactory.getLogger(SecureBulkLoadManager.class);<a name="line.100"></a>
+<span class="sourceLineNo">101</span><a name="line.101"></a>
+<span class="sourceLineNo">102</span>  private final static FsPermission PERM_ALL_ACCESS = FsPermission.valueOf("-rwxrwxrwx");<a name="line.102"></a>
+<span class="sourceLineNo">103</span>  private final static FsPermission PERM_HIDDEN = FsPermission.valueOf("-rwx--x--x");<a name="line.103"></a>
+<span class="sourceLineNo">104</span>  private SecureRandom random;<a name="line.104"></a>
+<span class="sourceLineNo">105</span>  private FileSystem fs;<a name="line.105"></a>
+<span class="sourceLineNo">106</span>  private Configuration conf;<a name="line.106"></a>
 <span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span>  private UserProvider userProvider;<a name="line.108"></a>
-<span class="sourceLineNo">109</span>  private Connection conn;<a name="line.109"></a>
-<span class="sourceLineNo">110</span><a name="line.110"></a>
-<span class="sourceLineNo">111</span>  SecureBulkLoadManager(Configuration conf, Connection conn) {<a name="line.111"></a>
-<span class="sourceLineNo">112</span>    this.conf = conf;<a name="line.112"></a>
-<span class="sourceLineNo">113</span>    this.conn = conn;<a name="line.113"></a>
-<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
+<span class="sourceLineNo">108</span>  //two levels so it doesn't get deleted accidentally<a name="line.108"></a>
+<span class="sourceLineNo">109</span>  //no sticky bit in Hadoop 1.0<a name="line.109"></a>
+<span class="sourceLineNo">110</span>  private Path baseStagingDir;<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span>  private UserProvider userProvider;<a name="line.112"></a>
+<span class="sourceLineNo">113</span>  private ConcurrentHashMap&lt;UserGroupInformation, Integer&gt; ugiReferenceCounter;<a name="line.113"></a>
+<span class="sourceLineNo">114</span>  private Connection conn;<a name="line.114"></a>
 <span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span>  public void start() throws IOException {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>    random = new SecureRandom();<a name="line.117"></a>
-<span class="sourceLineNo">118</span>    userProvider = UserProvider.instantiate(conf);<a name="line.118"></a>
-<span class="sourceLineNo">119</span>    fs = FileSystem.get(conf);<a name="line.119"></a>
-<span class="sourceLineNo">120</span>    baseStagingDir = new Path(FSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);<a name="line.120"></a>
-<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span>    if (conf.get("hbase.bulkload.staging.dir") != null) {<a name="line.122"></a>
-<span class="sourceLineNo">123</span>      LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is "<a name="line.123"></a>
-<span class="sourceLineNo">124</span>          + baseStagingDir);<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    }<a name="line.125"></a>
-<span class="sourceLineNo">126</span>    if (!fs.exists(baseStagingDir)) {<a name="line.126"></a>
-<span class="sourceLineNo">127</span>      fs.mkdirs(baseStagingDir, PERM_HIDDEN);<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    }<a name="line.128"></a>
-<span class="sourceLineNo">129</span>  }<a name="line.129"></a>
-<span class="sourceLineNo">130</span><a name="line.130"></a>
-<span class="sourceLineNo">131</span>  public void stop() throws IOException {<a name="line.131"></a>
-<span class="sourceLineNo">132</span>  }<a name="line.132"></a>
-<span class="sourceLineNo">133</span><a name="line.133"></a>
-<span class="sourceLineNo">134</span>  public String prepareBulkLoad(final HRegion region, final PrepareBulkLoadRequest request)<a name="line.134"></a>
-<span class="sourceLineNo">135</span>      throws IOException {<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    User user = getActiveUser();<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    region.getCoprocessorHost().prePrepareBulkLoad(user);<a name="line.137"></a>
-<span class="sourceLineNo">138</span><a name="line.138"></a>
-<span class="sourceLineNo">139</span>    String bulkToken =<a name="line.139"></a>
-<span class="sourceLineNo">140</span>        createStagingDir(baseStagingDir, user, region.getTableDescriptor().getTableName())<a name="line.140"></a>
-<span class="sourceLineNo">141</span>            .toString();<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span>    return bulkToken;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>  }<a name="line.144"></a>
-<span class="sourceLineNo">145</span><a name="line.145"></a>
-<span class="sourceLineNo">146</span>  public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request)<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      throws IOException {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    try {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser());<a name="line.149"></a>
-<span class="sourceLineNo">150</span><a name="line.150"></a>
-<span class="sourceLineNo">151</span>      Path path = new Path(request.getBulkToken());<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      if (!fs.delete(path, true)) {<a name="line.152"></a>
-<span class="sourceLineNo">153</span>        if (fs.exists(path)) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>          throw new IOException("Failed to clean up " + path);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>        }<a name="line.155"></a>
-<span class="sourceLineNo">156</span>      }<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      LOG.info("Cleaned up " + path + " successfully.");<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    } finally {<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      UserGroupInformation ugi = getActiveUser().getUGI();<a name="line.159"></a>
-<span class="sourceLineNo">160</span>      try {<a name="line.160"></a>
-<span class="sourceLineNo">161</span>        if (!UserGroupInformation.getLoginUser().equals(ugi)) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span>          FileSystem.closeAllForUGI(ugi);<a name="line.162"></a>
-<span class="sourceLineNo">163</span>        }<a name="line.163"></a>
-<span class="sourceLineNo">164</span>      } catch (IOException e) {<a name="line.164"></a>
-<span class="sourceLineNo">165</span>        LOG.error("Failed to close FileSystem for: " + ugi, e);<a name="line.165"></a>
-<span class="sourceLineNo">166</span>      }<a name="line.166"></a>
-<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
-<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span>  public Map&lt;byte[], List&lt;Path&gt;&gt; secureBulkLoadHFiles(final HRegion region,<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      final BulkLoadHFileRequest request) throws IOException {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    final List&lt;Pair&lt;byte[], String&gt;&gt; familyPaths = new ArrayList&lt;&gt;(request.getFamilyPathCount());<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {<a name="line.173"></a>
-<span class="sourceLineNo">174</span>      familyPaths.add(new Pair&lt;&gt;(el.getFamily().toByteArray(), el.getPath()));<a name="line.174"></a>
-<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
-<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span>    Token userToken = null;<a name="line.177"></a>
-<span class="sourceLineNo">178</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span>      userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()<a name="line.179"></a>
-<span class="sourceLineNo">180</span>              .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(<a name="line.180"></a>
-<span class="sourceLineNo">181</span>              request.getFsToken().getService()));<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    }<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    final String bulkToken = request.getBulkToken();<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    User user = getActiveUser();<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    final UserGroupInformation ugi = user.getUGI();<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      try {<a name="line.187"></a>
-<span class="sourceLineNo">188</span>        Token tok = TokenUtil.obtainToken(conn);<a name="line.188"></a>
-<span class="sourceLineNo">189</span>        if (tok != null) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span>          boolean b = ugi.addToken(tok);<a name="line.190"></a>
-<span class="sourceLineNo">191</span>          LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>        }<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      } catch (IOException ioe) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span>        LOG.warn("unable to add token", ioe);<a name="line.194"></a>
-<span class="sourceLineNo">195</span>      }<a name="line.195"></a>
-<span class="sourceLineNo">196</span>    }<a name="line.196"></a>
-<span class="sourceLineNo">197</span>    if (userToken != null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>      ugi.addToken(userToken);<a name="line.198"></a>
-<span class="sourceLineNo">199</span>    } else if (userProvider.isHadoopSecurityEnabled()) {<a name="line.199"></a>
-<span class="sourceLineNo">200</span>      //we allow this to pass through in "simple" security mode<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      //for mini cluster testing<a name="line.201"></a>
-<span class="sourceLineNo">202</span>      throw new DoNotRetryIOException("User token cannot be null");<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span>    if (region.getCoprocessorHost() != null) {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>      region.getCoprocessorHost().preBulkLoadHFile(familyPaths);<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    }<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    Map&lt;byte[], List&lt;Path&gt;&gt; map = null;<a name="line.208"></a>
-<span class="sourceLineNo">209</span><a name="line.209"></a>
-<span class="sourceLineNo">210</span>    try {<a name="line.210"></a>
-<span class="sourceLineNo">211</span>      // Get the target fs (HBase region server fs) delegation token<a name="line.211"></a>
-<span class="sourceLineNo">212</span>      // Since we have checked the permission via 'preBulkLoadHFile', now let's give<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      // the 'request user' necessary token to operate on the target fs.<a name="line.213"></a>
-<span class="sourceLineNo">214</span>      // After this point the 'doAs' user will hold two tokens, one for the source fs<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      // ('request user'), another for the target fs (HBase region server principal).<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      if (userProvider.isHadoopSecurityEnabled()) {<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider,"renewer");<a name="line.217"></a>
-<span class="sourceLineNo">218</span>        targetfsDelegationToken.acquireDelegationToken(fs);<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span>        Token&lt;?&gt; targetFsToken = targetfsDelegationToken.getUserToken();<a name="line.220"></a>
-<span class="sourceLineNo">221</span>        if (targetFsToken != null<a name="line.221"></a>
-<span class="sourceLineNo">222</span>            &amp;&amp; (userToken == null || !targetFsToken.getService().equals(userToken.getService()))){<a name="line.222"></a>
-<span class="sourceLineNo">223</span>          ugi.addToken(targetFsToken);<a name="line.223"></a>
-<span class="sourceLineNo">224</span>        }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>      }<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>      map = ugi.doAs(new PrivilegedAction&lt;Map&lt;byte[], List&lt;Path&gt;&gt;&gt;() {<a name="line.227"></a>
-<span class="sourceLineNo">228</span>        @Override<a name="line.228"></a>
-<span class="sourceLineNo">229</span>        public Map&lt;byte[], List&lt;Path&gt;&gt; run() {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>          FileSystem fs = null;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>          try {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>            fs = FileSystem.get(conf);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>            for(Pair&lt;byte[], String&gt; el: familyPaths) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span>              Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));<a name="line.234"></a>
-<span class="sourceLineNo">235</span>              if(!fs.exists(stageFamily)) {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>                fs.mkdirs(stageFamily);<a name="line.236"></a>
-<span class="sourceLineNo">237</span>                fs.setPermission(stageFamily, PERM_ALL_ACCESS);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>              }<a name="line.238"></a>
-<span class="sourceLineNo">239</span>            }<a name="line.239"></a>
-<span class="sourceLineNo">240</span>            //We call bulkLoadHFiles as requesting user<a name="line.240"></a>
-<span class="sourceLineNo">241</span>            //To enable access prior to staging<a name="line.241"></a>
-<span class="sourceLineNo">242</span>            return region.bulkLoadHFiles(familyPaths, true,<a name="line.242"></a>
-<span class="sourceLineNo">243</span>                new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile());<a name="line.243"></a>
-<span class="sourceLineNo">244</span>          } catch (Exception e) {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>            LOG.error("Failed to complete bulk load", e);<a name="line.245"></a>
-<span class="sourceLineNo">246</span>          }<a name="line.246"></a>
-<span class="sourceLineNo">247</span>          return null;<a name="line.247"></a>
-<span class="sourceLineNo">248</span>        }<a name="line.248"></a>
-<span class="sourceLineNo">249</span>      });<a name="line.249"></a>
-<span class="sourceLineNo">250</span>    } finally {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>      if (region.getCoprocessorHost() != null) {<a name="line.251"></a>
-<span class="sourceLineNo">252</span>        region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map);<a name="line.252"></a>
-<span class="sourceLineNo">253</span>      }<a name="line.253"></a>
-<span class="sourceLineNo">254</span>    }<a name="line.254"></a>
-<span class="sourceLineNo">255</span>    return map;<a name="line.255"></a>
-<span class="sourceLineNo">256</span>  }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span>  private Path createStagingDir(Path baseDir,<a name="line.258"></a>
-<span class="sourceLineNo">259</span>                                User user,<a name="line.259"></a>
-<span class="sourceLineNo">260</span>                                TableName tableName) throws IOException {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>    String tblName = tableName.getNameAsString().replace(":", "_");<a name="line.261"></a>
-<span class="sourceLineNo">262</span>    String randomDir = user.getShortName()+"__"+ tblName +"__"+<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));<a name="line.263"></a>
-<span class="sourceLineNo">264</span>    return createStagingDir(baseDir, user, randomDir);<a name="line.264"></a>
-<span class="sourceLineNo">265</span>  }<a name="line.265"></a>
-<span class="sourceLineNo">266</span><a name="line.266"></a>
-<span class="sourceLineNo">267</span>  private Path createStagingDir(Path baseDir,<a name="line.267"></a>
-<span class="sourceLineNo">268</span>                                User user,<a name="line.268"></a>
-<span class="sourceLineNo">269</span>                                String randomDir) throws IOException {<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    Path p = new Path(baseDir, randomDir);<a name="line.270"></a>
-<span class="sourceLineNo">271</span>    fs.mkdirs(p, PERM_ALL_ACCESS);<a name="line.271"></a>
-<span class="sourceLineNo">272</span>    fs.setPermission(p, PERM_ALL_ACCESS);<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    return p;<a name="line.273"></a>
-<span class="sourceLineNo">274</span>  }<a name="line.274"></a>
-<span class="sourceLineNo">275</span><a name="line.275"></a>
-<span class="sourceLineNo">276</span>  private User getActiveUser() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>    // for non-rpc handling, fallback to system user<a name="line.277"></a>
-<span class="sourceLineNo">278</span>    User user = RpcServer.getRequestUser().orElse(userProvider.getCurrent());<a name="line.278"></a>
-<span class="sourceLineNo">279</span>    // this is for testing<a name="line.279"></a>
-<span class="sourceLineNo">280</span>    if (userProvider.isHadoopSecurityEnabled() &amp;&amp;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>        "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {<a name="line.281"></a>
-<span class="sourceLineNo">282</span>      return User.createUserForTesting(conf, user.getShortName(), new String[] {});<a name="line.282"></a>
-<span class="sourceLineNo">283</span>    }<a name="line.283"></a>
-<span class="sourceLineNo">284</span><a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return user;<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  private static class SecureBulkLoadListener implements BulkLoadListener {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    // Target filesystem<a name="line.289"></a>
-<span class="sourceLineNo">290</span>    private final FileSystem fs;<a name="line.290"></a>
-<span class="sourceLineNo">291</span>    private final String stagingDir;<a name="line.291"></a>
-<span class="sourceLineNo">292</span>    private final Configuration conf;<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    // Source filesystem<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    private FileSystem srcFs = null;<a name="line.294"></a>
-<span class="sourceLineNo">295</span>    private Map&lt;String, FsPermission&gt; origPermissions = null;<a name="line.295"></a>
-<span class="sourceLineNo">296</span><a name="line.296"></a>
-<span class="sourceLineNo">297</span>    public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {<a name="line.297"></a>
-<span class="sourceLineNo">298</span>      this.fs = fs;<a name="line.298"></a>
-<span class="sourceLineNo">299</span>      this.stagingDir = stagingDir;<a name="line.299"></a>
-<span class="sourceLineNo">300</span>      this.conf = conf;<a name="line.300"></a>
-<span class="sourceLineNo">301</span>      this.origPermissions = new HashMap&lt;&gt;();<a name="line.301"></a>
-<span class="sourceLineNo">302</span>    }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span>    @Override<a name="line.304"></a>
-<span class="sourceLineNo">305</span>    public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile)<a name="line.305"></a>
-<span class="sourceLineNo">306</span>        throws IOException {<a name="line.306"></a>
-<span class="sourceLineNo">307</span>      Path p = new Path(srcPath);<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.308"></a>
+<span class="sourceLineNo">116</span>  SecureBulkLoadManager(Configuration conf, Connection conn) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span>    this.conf = conf;<a name="line.117"></a>
+<span class="sourceLineNo">118</span>    this.conn = conn;<a name="line.118"></a>
+<span class="sourceLineNo">119</span>  }<a name="line.119"></a>
+<span class="sourceLineNo">120</span><a name="line.120"></a>
+<span class="sourceLineNo">121</span>  public void start() throws IOException {<a name="line.121"></a>
+<span class="sourceLineNo">122</span>    random = new SecureRandom();<a name="line.122"></a>
+<span class="sourceLineNo">123</span>    userProvider = UserProvider.instantiate(conf);<a name="line.123"></a>
+<span class="sourceLineNo">124</span>    ugiReferenceCounter = new ConcurrentHashMap&lt;&gt;();<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    fs = FileSystem.get(conf);<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    baseStagingDir = new Path(FSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>    if (conf.get("hbase.bulkload.staging.dir") != null) {<a name="line.128"></a>
+<span class="sourceLineNo">129</span>      LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is "<a name="line.129"></a>
+<span class="sourceLineNo">130</span>          + baseStagingDir);<a name="line.130"></a>
+<span class="sourceLineNo">131</span>    }<a name="line.131"></a>
+<span class="sourceLineNo">132</span>    if (!fs.exists(baseStagingDir)) {<a name="line.132"></a>
+<span class="sourceLineNo">133</span>      fs.mkdirs(baseStagingDir, PERM_HIDDEN);<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    }<a name="line.134"></a>
+<span class="sourceLineNo">135</span>  }<a name="line.135"></a>
+<span class="sourceLineNo">136</span><a name="line.136"></a>
+<span class="sourceLineNo">137</span>  public void stop() throws IOException {<a name="line.137"></a>
+<span class="sourceLineNo">138</span>  }<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span>  public String prepareBulkLoad(final HRegion region, final PrepareBulkLoadRequest request)<a name="line.140"></a>
+<span class="sourceLineNo">141</span>      throws IOException {<a name="line.141"></a>
+<span class="sourceLineNo">142</span>    User user = getActiveUser();<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    region.getCoprocessorHost().prePrepareBulkLoad(user);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span>    String bulkToken =<a name="line.145"></a>
+<span class="sourceLineNo">146</span>        createStagingDir(baseStagingDir, user, region.getTableDescriptor().getTableName())<a name="line.146"></a>
+<span class="sourceLineNo">147</span>            .toString();<a name="line.147"></a>
+<span class="sourceLineNo">148</span><a name="line.148"></a>
+<span class="sourceLineNo">149</span>    return bulkToken;<a name="line.149"></a>
+<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span>  public void cleanupBulkLoad(final HRegion region, final CleanupBulkLoadRequest request)<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    try {<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      region.getCoprocessorHost().preCleanupBulkLoad(getActiveUser());<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span>      Path path = new Path(request.getBulkToken());<a name="line.157"></a>
+<span class="sourceLineNo">158</span>      if (!fs.delete(path, true)) {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>        if (fs.exists(path)) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>          throw new IOException("Failed to clean up " + path);<a name="line.160"></a>
+<span class="sourceLineNo">161</span>        }<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      }<a name="line.162"></a>
+<span class="sourceLineNo">163</span>      LOG.info("Cleaned up " + path + " successfully.");<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    } finally {<a name="line.164"></a>
+<span class="sourceLineNo">165</span>      UserGroupInformation ugi = getActiveUser().getUGI();<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      try {<a name="line.166"></a>
+<span class="sourceLineNo">167</span>        if (!UserGroupInformation.getLoginUser().equals(ugi) &amp;&amp; !isUserReferenced(ugi)) {<a name="line.167"></a>
+<span class="sourceLineNo">168</span>          FileSystem.closeAllForUGI(ugi);<a name="line.168"></a>
+<span class="sourceLineNo">169</span>        }<a name="line.169"></a>
+<span class="sourceLineNo">170</span>      } catch (IOException e) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>        LOG.error("Failed to close FileSystem for: " + ugi, e);<a name="line.171"></a>
+<span class="sourceLineNo">172</span>      }<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    }<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  }<a name="line.174"></a>
+<span class="sourceLineNo">175</span><a name="line.175"></a>
+<span class="sourceLineNo">176</span>  private Consumer&lt;HRegion&gt; fsCreatedListener;<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  @VisibleForTesting<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  void setFsCreatedListener(Consumer&lt;HRegion&gt; fsCreatedListener) {<a name="line.179"></a>
+<span class="sourceLineNo">180</span>    this.fsCreatedListener = fsCreatedListener;<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  }<a name="line.181"></a>
+<span class="sourceLineNo">182</span><a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span>  private void incrementUgiReference(UserGroupInformation ugi) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>    ugiReferenceCounter.merge(ugi, 1, new BiFunction&lt;Integer, Integer, Integer&gt;() {<a name="line.185"></a>
+<span class="sourceLineNo">186</span>      @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span>      public Integer apply(Integer oldvalue, Integer value) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span>        return ++oldvalue;<a name="line.188"></a>
+<span class="sourceLineNo">189</span>      }<a name="line.189"></a>
+<span class="sourceLineNo">190</span>    });<a name="line.190"></a>
+<span class="sourceLineNo">191</span>  }<a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span>  private void decrementUgiReference(UserGroupInformation ugi) {<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    ugiReferenceCounter.computeIfPresent(ugi,<a name="line.194"></a>
+<span class="sourceLineNo">195</span>        new BiFunction&lt;UserGroupInformation, Integer, Integer&gt;() {<a name="line.195"></a>
+<span class="sourceLineNo">196</span>          @Override<a name="line.196"></a>
+<span class="sourceLineNo">197</span>          public Integer apply(UserGroupInformation key, Integer value) {<a name="line.197"></a>
+<span class="sourceLineNo">198</span>            return value &gt; 1 ? --value : null;<a name="line.198"></a>
+<span class="sourceLineNo">199</span>          }<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      });<a name="line.200"></a>
+<span class="sourceLineNo">201</span>  }<a name="line.201"></a>
+<span class="sourceLineNo">202</span><a name="line.202"></a>
+<span class="sourceLineNo">203</span>  private boolean isUserReferenced(UserGroupInformation ugi) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>    Integer count = ugiReferenceCounter.get(ugi);<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    return count != null &amp;&amp; count &gt; 0;<a name="line.205"></a>
+<span class="sourceLineNo">206</span>  }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span>  public Map&lt;byte[], List&lt;Path&gt;&gt; secureBulkLoadHFiles(final HRegion region,<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      final BulkLoadHFileRequest request) throws IOException {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    final List&lt;Pair&lt;byte[], String&gt;&gt; familyPaths = new ArrayList&lt;&gt;(request.getFamilyPathCount());<a name="line.210"></a>
+<span class="sourceLineNo">211</span>    for(ClientProtos.BulkLoadHFileRequest.FamilyPath el : request.getFamilyPathList()) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>      familyPaths.add(new Pair&lt;&gt;(el.getFamily().toByteArray(), el.getPath()));<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    }<a name="line.213"></a>
+<span class="sourceLineNo">214</span><a name="line.214"></a>
+<span class="sourceLineNo">215</span>    Token userToken = null;<a name="line.215"></a>
+<span class="sourceLineNo">216</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.216"></a>
+<span class="sourceLineNo">217</span>      userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()<a name="line.217"></a>
+<span class="sourceLineNo">218</span>              .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(<a name="line.218"></a>
+<span class="sourceLineNo">219</span>              request.getFsToken().getService()));<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    final String bulkToken = request.getBulkToken();<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    User user = getActiveUser();<a name="line.222"></a>
+<span class="sourceLineNo">223</span>    final UserGroupInformation ugi = user.getUGI();<a name="line.223"></a>
+<span class="sourceLineNo">224</span>    if (userProvider.isHadoopSecurityEnabled()) {<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      try {<a name="line.225"></a>
+<span class="sourceLineNo">226</span>        Token tok = TokenUtil.obtainToken(conn);<a name="line.226"></a>
+<span class="sourceLineNo">227</span>        if (tok != null) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span>          boolean b = ugi.addToken(tok);<a name="line.228"></a>
+<span class="sourceLineNo">229</span>          LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);<a name="line.229"></a>
+<span class="sourceLineNo">230</span>        }<a name="line.230"></a>
+<span class="sourceLineNo">231</span>      } catch (IOException ioe) {<a name="line.231"></a>
+<span class="sourceLineNo">232</span>        LOG.warn("unable to add token", ioe);<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      }<a name="line.233"></a>
+<span class="sourceLineNo">234</span>    }<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    if (userToken != null) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span>      ugi.addToken(userToken);<a name="line.236"></a>
+<span class="sourceLineNo">237</span>    } else if (userProvider.isHadoopSecurityEnabled()) {<a name="line.237"></a>
+<span class="sourceLineNo">238</span>      //we allow this to pass through in "simple" security mode<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      //for mini cluster testing<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      throw new DoNotRetryIOException("User token cannot be null");<a name="line.240"></a>
+<span class="sourceLineNo">241</span>    }<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span>    if (region.getCoprocessorHost() != null) {<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      region.getCoprocessorHost().preBulkLoadHFile(familyPaths);<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span>    Map&lt;byte[], List&lt;Path&gt;&gt; map = null;<a name="line.246"></a>
+<span class="sourceLineNo">247</span><a name="line.247"></a>
+<span class="sourceLineNo">248</span>    try {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      incrementUgiReference(ugi);<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      // Get the target fs (HBase region server fs) delegation token<a name="line.250"></a>
+<span class="sourceLineNo">251</span>      // Since we have checked the permission via 'preBulkLoadHFile', now let's give<a name="line.251"></a>
+<span class="sourceLineNo">252</span>      // the 'request user' necessary token to operate on the target fs.<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      // After this point the 'doAs' user will hold two tokens, one for the source fs<a name="line.253"></a>
+<span class="sourceLineNo">254</span>      // ('request user'), another for the target fs (HBase region server principal).<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      if (userProvider.isHadoopSecurityEnabled()) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span>        FsDelegationToken targetfsDelegationToken = new FsDelegationToken(userProvider,"renewer");<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        targetfsDelegationToken.acquireDelegationToken(fs);<a name="line.257"></a>
+<span class="sourceLineNo">258</span><a name="line.258"></a>
+<span class="sourceLineNo">259</span>        Token&lt;?&gt; targetFsToken = targetfsDelegationToken.getUserToken();<a name="line.259"></a>
+<span class="sourceLineNo">260</span>        if (targetFsToken != null<a name="line.260"></a>
+<span class="sourceLineNo">261</span>            &amp;&amp; (userToken == null || !targetFsToken.getService().equals(userToken.getService()))){<a name="line.261"></a>
+<span class="sourceLineNo">262</span>          ugi.addToken(targetFsToken);<a name="line.262"></a>
+<span class="sourceLineNo">263</span>        }<a name="line.263"></a>
+<span class="sourceLineNo">264</span>      }<a name="line.264"></a>
+<span class="sourceLineNo">265</span><a name="line.265"></a>
+<span class="sourceLineNo">266</span>      map = ugi.doAs(new PrivilegedAction&lt;Map&lt;byte[], List&lt;Path&gt;&gt;&gt;() {<a name="line.266"></a>
+<span class="sourceLineNo">267</span>        @Override<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        public Map&lt;byte[], List&lt;Path&gt;&gt; run() {<a name="line.268"></a>
+<span class="sourceLineNo">269</span>          FileSystem fs = null;<a name="line.269"></a>
+<span class="sourceLineNo">270</span>          try {<a name="line.270"></a>
+<span class="sourceLineNo">271</span>            fs = FileSystem.get(conf);<a name="line.271"></a>
+<span class="sourceLineNo">272</span>            for(Pair&lt;byte[], String&gt; el: familyPaths) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span>              Path stageFamily = new Path(bulkToken, Bytes.toString(el.getFirst()));<a name="line.273"></a>
+<span class="sourceLineNo">274</span>              if(!fs.exists(stageFamily)) {<a name="line.274"></a>
+<span class="sourceLineNo">275</span>                fs.mkdirs(stageFamily);<a name="line.275"></a>
+<span class="sourceLineNo">276</span>                fs.setPermission(stageFamily, PERM_ALL_ACCESS);<a name="line.276"></a>
+<span class="sourceLineNo">277</span>              }<a name="line.277"></a>
+<span class="sourceLineNo">278</span>            }<a name="line.278"></a>
+<span class="sourceLineNo">279</span>            if (fsCreatedListener != null) {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>              fsCreatedListener.accept(region);<a name="line.280"></a>
+<span class="sourceLineNo">281</span>            }<a name="line.281"></a>
+<span class="sourceLineNo">282</span>            //We call bulkLoadHFiles as requesting user<a name="line.282"></a>
+<span class="sourceLineNo">283</span>            //To enable access prior to staging<a name="line.283"></a>
+<span class="sourceLineNo">284</span>            return region.bulkLoadHFiles(familyPaths, true,<a name="line.284"></a>
+<span class="sourceLineNo">285</span>                new SecureBulkLoadListener(fs, bulkToken, conf), request.getCopyFile());<a name="line.285"></a>
+<span class="sourceLineNo">286</span>          } catch (Exception e) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>            LOG.error("Failed to complete bulk load", e);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>          }<a name="line.288"></a>
+<span class="sourceLineNo">289</span>          return null;<a name="line.289"></a>
+<span class="sourceLineNo">290</span>        }<a name="line.290"></a>
+<span class="sourceLineNo">291</span>      });<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    } finally {<a name="line.292"></a>
+<span class="sourceLineNo">293</span>      decrementUgiReference(ugi);<a name="line.293"></a>
+<span class="sourceLineNo">294</span>      if (region.getCoprocessorHost() != null) {<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        region.getCoprocessorHost().postBulkLoadHFile(familyPaths, map);<a name="line.295"></a>
+<span class="sourceLineNo">296</span>      }<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    }<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    return map;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>  private Path createStagingDir(Path baseDir,<a name="line.301"></a>
+<span class="sourceLineNo">302</span>                                User user,<a name="line.302"></a>
+<span class="sourceLineNo">303</span>                                TableName tableName) throws IOException {<a name="line.303"></a>
+<span class="sourceLineNo">304</span>    String tblName = tableName.getNameAsString().replace(":", "_");<a name="line.304"></a>
+<span class="sourceLineNo">305</span>    String randomDir = user.getShortName()+"__"+ tblName +"__"+<a name="line.305"></a>
+<span class="sourceLineNo">306</span>        (new BigInteger(RANDOM_WIDTH, random).toString(RANDOM_RADIX));<a name="line.306"></a>
+<span class="sourceLineNo">307</span>    return createStagingDir(baseDir, user, randomDir);<a name="line.307"></a>
+<span class="sourceLineNo">308</span>  }<a name="line.308"></a>
 <span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span>      // In case of Replication for bulk load files, hfiles are already copied in staging directory<a name="line.310"></a>
-<span class="sourceLineNo">311</span>      if (p.equals(stageP)) {<a name="line.311"></a>
-<span class="sourceLineNo">312</span>        LOG.debug(p.getName()<a name="line.312"></a>
-<span class="sourceLineNo">313</span>            + " is already available in staging directory. Skipping copy or rename.");<a name="line.313"></a>
-<span class="sourceLineNo">314</span>        return stageP.toString();<a name="line.314"></a>
-<span class="sourceLineNo">315</span>      }<a name="line.315"></a>
-<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span>      if (srcFs == null) {<a name="line.317"></a>
-<span class="sourceLineNo">318</span>        srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.318"></a>
-<span class="sourceLineNo">319</span>      }<a name="line.319"></a>
-<span class="sourceLineNo">320</span><a name="line.320"></a>
-<span class="sourceLineNo">321</span>      if(!isFile(p)) {<a name="line.321"></a>
-<span class="sourceLineNo">322</span>        throw new IOException("Path does not reference a file: " + p);<a name="line.322"></a>
-<span class="sourceLineNo">323</span>      }<a name="line.323"></a>
-<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span>      // Check to see if the source and target filesystems are the same<a name="line.325"></a>
-<span class="sourceLineNo">326</span>      if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span>        LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +<a name="line.327"></a>
-<span class="sourceLineNo">328</span>            "the destination filesystem. Copying file over to destination staging dir.");<a name="line.328"></a>
-<span class="sourceLineNo">329</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>      } else if (copyFile) {<a name="line.330"></a>
-<span class="sourceLineNo">331</span>        LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");<a name="line.331"></a>
-<span class="sourceLineNo">332</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.332"></a>
-<span class="sourceLineNo">333</span>      } else {<a name="line.333"></a>
-<span class="sourceLineNo">334</span>        LOG.debug("Moving " + p + " to " + stageP);<a name="line.334"></a>
-<span class="sourceLineNo">335</span>        FileStatus origFileStatus = fs.getFileStatus(p);<a name="line.335"></a>
-<span class="sourceLineNo">336</span>        origPermissions.put(srcPath, origFileStatus.getPermission());<a name="line.336"></a>
-<span class="sourceLineNo">337</span>        if(!fs.rename(p, stageP)) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span>          throw new IOException("Failed to move HFile: " + p + " to " + stageP);<a name="line.338"></a>
-<span class="sourceLineNo">339</span>        }<a name="line.339"></a>
-<span class="sourceLineNo">340</span>      }<a name="line.340"></a>
-<span class="sourceLineNo">341</span>      fs.setPermission(stageP, PERM_ALL_ACCESS);<a name="line.341"></a>
-<span class="sourceLineNo">342</span>      return stageP.toString();<a name="line.342"></a>
-<span class="sourceLineNo">343</span>    }<a name="line.343"></a>
-<span class="sourceLineNo">344</span><a name="line.344"></a>
-<span class="sourceLineNo">345</span>    @Override<a name="line.345"></a>
-<span class="sourceLineNo">346</span>    public void doneBulkLoad(byte[] family, String srcPath) throws IOException {<a name="line.346"></a>
-<span class="sourceLineNo">347</span>      LOG.debug("Bulk Load done for: " + srcPath);<a name="line.347"></a>
-<span class="sourceLineNo">348</span>      closeSrcFs();<a name="line.348"></a>
-<span class="sourceLineNo">349</span>    }<a name="line.349"></a>
-<span class="sourceLineNo">350</span><a name="line.350"></a>
-<span class="sourceLineNo">351</span>    private void closeSrcFs() throws IOException {<a name="line.351"></a>
-<span class="sourceLineNo">352</span>      if (srcFs != null) {<a name="line.352"></a>
-<span class="sourceLineNo">353</span>        srcFs.close();<a name="line.353"></a>
-<span class="sourceLineNo">354</span>        srcFs = null;<a name="line.354"></a>
-<span class="sourceLineNo">355</span>      }<a name="line.355"></a>
-<span class="sourceLineNo">356</span>    }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span>    @Override<a name="line.358"></a>
-<span class="sourceLineNo">359</span>    public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {<a name="line.359"></a>
-<span class="sourceLineNo">360</span>      try {<a name="line.360"></a>
-<span class="sourceLineNo">361</span>        Path p = new Path(srcPath);<a name="line.361"></a>
-<span class="sourceLineNo">362</span>        if (srcFs == null) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span>          srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.363"></a>
-<span class="sourceLineNo">364</span>        }<a name="line.364"></a>
-<span class="sourceLineNo">365</span>        if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.365"></a>
-<span class="sourceLineNo">366</span>          // files are copied so no need to move them back<a name="line.366"></a>
-<span class="sourceLineNo">367</span>          return;<a name="line.367"></a>
-<span class="sourceLineNo">368</span>        }<a name="line.368"></a>
-<span class="sourceLineNo">369</span>        Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span>        // In case of Replication for bulk load files, hfiles are not renamed by end point during<a name="line.371"></a>
-<span class="sourceLineNo">372</span>        // prepare stage, so no need of rename here again<a name="line.372"></a>
-<span class="sourceLineNo">373</span>        if (p.equals(stageP)) {<a name="line.373"></a>
-<span class="sourceLineNo">374</span>          LOG.debug(p.getName() + " is already available in source directory. Skipping rename.");<a name="line.374"></a>
-<span class="sourceLineNo">375</span>          return;<a name="line.375"></a>
-<span class="sourceLineNo">376</span>        }<a name="line.376"></a>
-<span class="sourceLineNo">377</span><a name="line.377"></a>
-<span class="sourceLineNo">378</span>        LOG.debug("Moving " + stageP + " back to " + p);<a name="line.378"></a>
-<span class="sourceLineNo">379</span>        if (!fs.rename(stageP, p)) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span>          throw new IOException("Failed to move HFile: " + stageP + " to " + p);<a name="line.380"></a>
-<span class="sourceLineNo">381</span>        }<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>        // restore original permission<a name="line.383"></a>
-<span class="sourceLineNo">384</span>        if (origPermissions.containsKey(srcPath)) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>          fs.setPermission(p, origPermissions.get(srcPath));<a name="line.385"></a>
-<span class="sourceLineNo">386</span>        } else {<a name="line.386"></a>
-<span class="sourceLineNo">387</span>          LOG.warn("Can't find previous permission for path=" + srcPath);<a name="line.387"></a>
-<span class="sourceLineNo">388</span>        }<a name="line.388"></a>
-<span class="sourceLineNo">389</span>      } finally {<a name="line.389"></a>
-<span class="sourceLineNo">390</span>        closeSrcFs();<a name="line.390"></a>
-<span class="sourceLineNo">391</span>      }<a name="line.391"></a>
+<span class="sourceLineNo">310</span>  private Path createStagingDir(Path baseDir,<a name="line.310"></a>
+<span class="sourceLineNo">311</span>                                User user,<a name="line.311"></a>
+<span class="sourceLineNo">312</span>                                String randomDir) throws IOException {<a name="line.312"></a>
+<span class="sourceLineNo">313</span>    Path p = new Path(baseDir, randomDir);<a name="line.313"></a>
+<span class="sourceLineNo">314</span>    fs.mkdirs(p, PERM_ALL_ACCESS);<a name="line.314"></a>
+<span class="sourceLineNo">315</span>    fs.setPermission(p, PERM_ALL_ACCESS);<a name="line.315"></a>
+<span class="sourceLineNo">316</span>    return p;<a name="line.316"></a>
+<span class="sourceLineNo">317</span>  }<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>  private User getActiveUser() throws IOException {<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    // for non-rpc handling, fallback to system user<a name="line.320"></a>
+<span class="sourceLineNo">321</span>    User user = RpcServer.getRequestUser().orElse(userProvider.getCurrent());<a name="line.321"></a>
+<span class="sourceLineNo">322</span>    // this is for testing<a name="line.322"></a>
+<span class="sourceLineNo">323</span>    if (userProvider.isHadoopSecurityEnabled() &amp;&amp;<a name="line.323"></a>
+<span class="sourceLineNo">324</span>        "simple".equalsIgnoreCase(conf.get(User.HBASE_SECURITY_CONF_KEY))) {<a name="line.324"></a>
+<span class="sourceLineNo">325</span>      return User.createUserForTesting(conf, user.getShortName(), new String[] {});<a name="line.325"></a>
+<span class="sourceLineNo">326</span>    }<a name="line.326"></a>
+<span class="sourceLineNo">327</span><a name="line.327"></a>
+<span class="sourceLineNo">328</span>    return user;<a name="line.328"></a>
+<span class="sourceLineNo">329</span>  }<a name="line.329"></a>
+<span class="sourceLineNo">330</span><a name="line.330"></a>
+<span class="sourceLineNo">331</span>  private static class SecureBulkLoadListener implements BulkLoadListener {<a name="line.331"></a>
+<span class="sourceLineNo">332</span>    // Target filesystem<a name="line.332"></a>
+<span class="sourceLineNo">333</span>    private final FileSystem fs;<a name="line.333"></a>
+<span class="sourceLineNo">334</span>    private final String stagingDir;<a name="line.334"></a>
+<span class="sourceLineNo">335</span>    private final Configuration conf;<a name="line.335"></a>
+<span class="sourceLineNo">336</span>    // Source filesystem<a name="line.336"></a>
+<span class="sourceLineNo">337</span>    private FileSystem srcFs = null;<a name="line.337"></a>
+<span class="sourceLineNo">338</span>    private Map&lt;String, FsPermission&gt; origPermissions = null;<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span>    public SecureBulkLoadListener(FileSystem fs, String stagingDir, Configuration conf) {<a name="line.340"></a>
+<span class="sourceLineNo">341</span>      this.fs = fs;<a name="line.341"></a>
+<span class="sourceLineNo">342</span>      this.stagingDir = stagingDir;<a name="line.342"></a>
+<span class="sourceLineNo">343</span>      this.conf = conf;<a name="line.343"></a>
+<span class="sourceLineNo">344</span>      this.origPermissions = new HashMap&lt;&gt;();<a name="line.344"></a>
+<span class="sourceLineNo">345</span>    }<a name="line.345"></a>
+<span class="sourceLineNo">346</span><a name="line.346"></a>
+<span class="sourceLineNo">347</span>    @Override<a name="line.347"></a>
+<span class="sourceLineNo">348</span>    public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile)<a name="line.348"></a>
+<span class="sourceLineNo">349</span>        throws IOException {<a name="line.349"></a>
+<span class="sourceLineNo">350</span>      Path p = new Path(srcPath);<a name="line.350"></a>
+<span class="sourceLineNo">351</span>      Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.351"></a>
+<span class="sourceLineNo">352</span><a name="line.352"></a>
+<span class="sourceLineNo">353</span>      // In case of Replication for bulk load files, hfiles are already copied in staging directory<a name="line.353"></a>
+<span class="sourceLineNo">354</span>      if (p.equals(stageP)) {<a name="line.354"></a>
+<span class="sourceLineNo">355</span>        LOG.debug(p.getName()<a name="line.355"></a>
+<span class="sourceLineNo">356</span>            + " is already available in staging directory. Skipping copy or rename.");<a name="line.356"></a>
+<span class="sourceLineNo">357</span>        return stageP.toString();<a name="line.357"></a>
+<span class="sourceLineNo">358</span>      }<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span>      if (srcFs == null) {<a name="line.360"></a>
+<span class="sourceLineNo">361</span>        srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.361"></a>
+<span class="sourceLineNo">362</span>      }<a name="line.362"></a>
+<span class="sourceLineNo">363</span><a name="line.363"></a>
+<span class="sourceLineNo">364</span>      if(!isFile(p)) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span>        throw new IOException("Path does not reference a file: " + p);<a name="line.365"></a>
+<span class="sourceLineNo">366</span>      }<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span>      // Check to see if the source and target filesystems are the same<a name="line.368"></a>
+<span class="sourceLineNo">369</span>      if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.369"></a>
+<span class="sourceLineNo">370</span>        LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +<a name="line.370"></a>
+<span class="sourceLineNo">371</span>            "the destination filesystem. Copying file over to destination staging dir.");<a name="line.371"></a>
+<span class="sourceLineNo">372</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.372"></a>
+<span class="sourceLineNo">373</span>      } else if (copyFile) {<a name="line.373"></a>
+<span class="sourceLineNo">374</span>        LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");<a name="line.374"></a>
+<span class="sourceLineNo">375</span>        FileUtil.copy(srcFs, p, fs, stageP, false, conf);<a name="line.375"></a>
+<span class="sourceLineNo">376</span>      } else {<a name="line.376"></a>
+<span class="sourceLineNo">377</span>        LOG.debug("Moving " + p + " to " + stageP);<a name="line.377"></a>
+<span class="sourceLineNo">378</span>        FileStatus origFileStatus = fs.getFileStatus(p);<a name="line.378"></a>
+<span class="sourceLineNo">379</span>        origPermissions.put(srcPath, origFileStatus.getPermission());<a name="line.379"></a>
+<span class="sourceLineNo">380</span>        if(!fs.rename(p, stageP)) {<a name="line.380"></a>
+<span class="sourceLineNo">381</span>          throw new IOException("Failed to move HFile: " + p + " to " + stageP);<a name="line.381"></a>
+<span class="sourceLineNo">382</span>        }<a name="line.382"></a>
+<span class="sourceLineNo">383</span>      }<a name="line.383"></a>
+<span class="sourceLineNo">384</span>      fs.setPermission(stageP, PERM_ALL_ACCESS);<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      return stageP.toString();<a name="line.385"></a>
+<span class="sourceLineNo">386</span>    }<a name="line.386"></a>
+<span class="sourceLineNo">387</span><a name="line.387"></a>
+<span class="sourceLineNo">388</span>    @Override<a name="line.388"></a>
+<span class="sourceLineNo">389</span>    public void doneBulkLoad(byte[] family, String srcPath) throws IOException {<a name="line.389"></a>
+<span class="sourceLineNo">390</span>      LOG.debug("Bulk Load done for: " + srcPath);<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      closeSrcFs();<a name="line.391"></a>
 <span class="sourceLineNo">392</span>    }<a name="line.392"></a>
 <span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    /**<a name="line.394"></a>
-<span class="sourceLineNo">395</span>     * Check if the path is referencing a file.<a name="line.395"></a>
-<span class="sourceLineNo">396</span>     * This is mainly needed to avoid symlinks.<a name="line.396"></a>
-<span class="sourceLineNo">397</span>     * @param p<a name="line.397"></a>
-<span class="sourceLineNo">398</span>     * @return true if the p is a file<a name="line.398"></a>
-<span class="sourceLineNo">399</span>     * @throws IOException<a name="line.399"></a>
-<span class="sourceLineNo">400</span>     */<a name="line.400"></a>
-<span class="sourceLineNo">401</span>    private boolean isFile(Path p) throws IOException {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      FileStatus status = srcFs.getFileStatus(p);<a name="line.402"></a>
-<span class="sourceLineNo">403</span>      boolean isFile = !status.isDirectory();<a name="line.403"></a>
-<span class="sourceLineNo">404</span>      try {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>        isFile = isFile &amp;&amp; !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);<a name="line.405"></a>
-<span class="sourceLineNo">406</span>      } catch (Exception e) {<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
-<span class="sourceLineNo">408</span>      return isFile;<a name="line.408"></a>
-<span class="sourceLineNo">409</span>    }<a name="line.409"></a>
-<span class="sourceLineNo">410</span>  }<a name="line.410"></a>
-<span class="sourceLineNo">411</span>}<a name="line.411"></a>
+<span class="sourceLineNo">394</span>    private void closeSrcFs() throws IOException {<a name="line.394"></a>
+<span class="sourceLineNo">395</span>      if (srcFs != null) {<a name="line.395"></a>
+<span class="sourceLineNo">396</span>        srcFs.close();<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        srcFs = null;<a name="line.397"></a>
+<span class="sourceLineNo">398</span>      }<a name="line.398"></a>
+<span class="sourceLineNo">399</span>    }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span>    @Override<a name="line.401"></a>
+<span class="sourceLineNo">402</span>    public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      try {<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        Path p = new Path(srcPath);<a name="line.404"></a>
+<span class="sourceLineNo">405</span>        if (srcFs == null) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>          srcFs = FileSystem.newInstance(p.toUri(), conf);<a name="line.406"></a>
+<span class="sourceLineNo">407</span>        }<a name="line.407"></a>
+<span class="sourceLineNo">408</span>        if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {<a name="line.408"></a>
+<span class="sourceLineNo">409</span>          // files are copied so no need to move them back<a name="line.409"></a>
+<span class="sourceLineNo">410</span>          return;<a name="line.410"></a>
+<span class="sourceLineNo">411</span>        }<a name="line.411"></a>
+<span class="sourceLineNo">412</span>        Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>        // In case of Replication for bulk load files, hfiles are not renamed by end point during<a name="line.414"></a>
+<span class="sourceLineNo">415</span>        // prepare stage, so no need of rename here again<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        if (p.equals(stageP)) {<a name="line.416"></a>
+<span class="sourceLineNo">417</span>          LOG.debug(p.getName() + " is already available in source directory. Skipping rename.");<a name="line.417"></a>
+<span class="sourceLineNo">418</span>          return;<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        }<a name="line.419"></a>
+<span class="sourceLineNo">420</span><a name="line.420"></a>
+<span class="sourceLineNo">421</span>        LOG.debug("Moving " + stageP + " back to " + p);<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        if (!fs.rename(stageP, p)) {<a name="line.422"></a>
+<span class="sourceLineNo">423</span>          throw new IOException("Failed to move HFile: " + stageP + " to " + p);<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        }<a name="line.424"></a>
+<span class="sourceLineNo">425</span><a name="line.425"></a>
+<span class="sourceLineNo">426</span>        // restore original permission<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        if (origPermissions.containsKey(srcPath)) {<a name="line.427"></a>
+<span class="sourceLineNo">428</span>          fs.setPermission(p, origPermissions.get(srcPath));<a name="line.428"></a>
+<span class="sourceLineNo">429</span>        } else {<a name="line.429"></a>
+<span class="sourceLineNo">430</span>          LOG.warn("Can't find previous permission for path=" + srcPath);<a name="line.430"></a>
+<span class="sourceLineNo">431</span>        }<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      } finally {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        closeSrcFs();<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span><a name="line.436"></a>
+<span class="sourceLineNo">437</span>    /**<a name="line.437"></a>
+<span class="sourceLineNo">438</span>     * Check if the path is referencing a file.<a name="line.438"></a>
+<span class="sourceLineNo">439</span>     * This is mainly needed to avoid symlinks.<a name="line.439"></a>
+<span class="sourceLineNo">440</span>     * @param p<a name="line.440"></a>
+<span class="sourceLineNo">441</span>     * @return true if the p is a file<a name="line.441"></a>
+<span class="sourceLineNo">442</span>     * @throws IOException<a name="line.442"></a>
+<span class="sourceLineNo">443</span>     */<a name="line.443"></a>
+<span class="sourceLineNo">444</span>    private boolean isFile(Path p) throws IOException {<a name="line.444"></a>
+<span class="sourceLineNo">445</span>      FileStatus status = srcFs.getFileStatus(p);<a name="line.445"></a>
+<span class="sourceLineNo">446</span>      boolean isFile = !status.isDirectory();<a name="line.446"></a>
+<span class="sourceLineNo">447</span>      try {<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        isFile = isFile &amp;&amp; !(Boolean)Methods.call(FileStatus.class, status, "isSymlink", null, null);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      } catch (Exception e) {<a name="line.449"></a>
+<span class="sourceLineNo">450</span>      }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>      return isFile;<a name="line.451"></a>
+<span class="sourceLineNo">452</span>    }<a name="line.452"></a>
+<span class="sourceLineNo">453</span>  }<a name="line.453"></a>
+<span class="sourceLineNo">454</span>}<a name="line.454"></a>
 
 
 


[11/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
index be5c3fc..839d12b 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/BitSetNode.html
@@ -139,294 +139,296 @@
 <span class="sourceLineNo">131</span><a name="line.131"></a>
 <span class="sourceLineNo">132</span>  public BitSetNode(BitSetNode other, boolean resetDelete) {<a name="line.132"></a>
 <span class="sourceLineNo">133</span>    this.start = other.start;<a name="line.133"></a>
-<span class="sourceLineNo">134</span>    this.partial = other.partial;<a name="line.134"></a>
-<span class="sourceLineNo">135</span>    this.modified = other.modified.clone();<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    // The resetDelete will be set to true when building cleanup tracker.<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    // The intention here is that, if a procedure is not modified in this tracker, then we do not<a name="line.137"></a>
-<span class="sourceLineNo">138</span>    // need to take care of it, so we will set deleted to true for these bits, i.e, if modified is<a name="line.138"></a>
-<span class="sourceLineNo">139</span>    // 0, then we set deleted to 1, otherwise keep it as is. So here, the equation is<a name="line.139"></a>
-<span class="sourceLineNo">140</span>    // deleted |= ~modified, i.e,<a name="line.140"></a>
-<span class="sourceLineNo">141</span>    if (resetDelete) {<a name="line.141"></a>
-<span class="sourceLineNo">142</span>      this.deleted = new long[other.deleted.length];<a name="line.142"></a>
-<span class="sourceLineNo">143</span>      for (int i = 0; i &lt; this.deleted.length; ++i) {<a name="line.143"></a>
-<span class="sourceLineNo">144</span>        this.deleted[i] |= ~(other.modified[i]);<a name="line.144"></a>
-<span class="sourceLineNo">145</span>      }<a name="line.145"></a>
-<span class="sourceLineNo">146</span>    } else {<a name="line.146"></a>
-<span class="sourceLineNo">147</span>      this.deleted = other.deleted.clone();<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
-<span class="sourceLineNo">149</span>  }<a name="line.149"></a>
-<span class="sourceLineNo">150</span><a name="line.150"></a>
-<span class="sourceLineNo">151</span>  public void insertOrUpdate(final long procId) {<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    updateState(procId, false);<a name="line.152"></a>
-<span class="sourceLineNo">153</span>  }<a name="line.153"></a>
-<span class="sourceLineNo">154</span><a name="line.154"></a>
-<span class="sourceLineNo">155</span>  public void delete(final long procId) {<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    updateState(procId, true);<a name="line.156"></a>
-<span class="sourceLineNo">157</span>  }<a name="line.157"></a>
-<span class="sourceLineNo">158</span><a name="line.158"></a>
-<span class="sourceLineNo">159</span>  public long getStart() {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    return start;<a name="line.160"></a>
-<span class="sourceLineNo">161</span>  }<a name="line.161"></a>
-<span class="sourceLineNo">162</span><a name="line.162"></a>
-<span class="sourceLineNo">163</span>  public long getEnd() {<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    return start + (modified.length &lt;&lt; ADDRESS_BITS_PER_WORD) - 1;<a name="line.164"></a>
-<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span>  public boolean contains(final long procId) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    return start &lt;= procId &amp;&amp; procId &lt;= getEnd();<a name="line.168"></a>
-<span class="sourceLineNo">169</span>  }<a name="line.169"></a>
-<span class="sourceLineNo">170</span><a name="line.170"></a>
-<span class="sourceLineNo">171</span>  public DeleteState isDeleted(final long procId) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.172"></a>
-<span class="sourceLineNo">173</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.173"></a>
-<span class="sourceLineNo">174</span>    if (wordIndex &gt;= deleted.length) {<a name="line.174"></a>
-<span class="sourceLineNo">175</span>      return DeleteState.MAYBE;<a name="line.175"></a>
-<span class="sourceLineNo">176</span>    }<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    return (deleted[wordIndex] &amp; (1L &lt;&lt; bitmapIndex)) != 0 ? DeleteState.YES : DeleteState.NO;<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  }<a name="line.178"></a>
-<span class="sourceLineNo">179</span><a name="line.179"></a>
-<span class="sourceLineNo">180</span>  public boolean isModified(long procId) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.181"></a>
-<span class="sourceLineNo">182</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.182"></a>
-<span class="sourceLineNo">183</span>    if (wordIndex &gt;= modified.length) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span>      return false;<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    }<a name="line.185"></a>
-<span class="sourceLineNo">186</span>    return (modified[wordIndex] &amp; (1L &lt;&lt; bitmapIndex)) != 0;<a name="line.186"></a>
-<span class="sourceLineNo">187</span>  }<a name="line.187"></a>
-<span class="sourceLineNo">188</span><a name="line.188"></a>
-<span class="sourceLineNo">189</span>  /**<a name="line.189"></a>
-<span class="sourceLineNo">190</span>   * @return true, if all the procedures has been modified.<a name="line.190"></a>
-<span class="sourceLineNo">191</span>   */<a name="line.191"></a>
-<span class="sourceLineNo">192</span>  public boolean isAllModified() {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>    // TODO: cache the value<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span>      if ((modified[i] | deleted[i]) != WORD_MASK) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>        return false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>    }<a name="line.198"></a>
-<span class="sourceLineNo">199</span>    return true;<a name="line.199"></a>
-<span class="sourceLineNo">200</span>  }<a name="line.200"></a>
-<span class="sourceLineNo">201</span><a name="line.201"></a>
-<span class="sourceLineNo">202</span>  /**<a name="line.202"></a>
-<span class="sourceLineNo">203</span>   * @return all the active procedure ids in this bit set.<a name="line.203"></a>
-<span class="sourceLineNo">204</span>   */<a name="line.204"></a>
-<span class="sourceLineNo">205</span>  public long[] getActiveProcIds() {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    List&lt;Long&gt; procIds = new ArrayList&lt;&gt;();<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    for (int wordIndex = 0; wordIndex &lt; modified.length; wordIndex++) {<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      if (deleted[wordIndex] == WORD_MASK || modified[wordIndex] == 0) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>        // This should be the common case, where most procedures has been deleted.<a name="line.209"></a>
-<span class="sourceLineNo">210</span>        continue;<a name="line.210"></a>
-<span class="sourceLineNo">211</span>      }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>      long baseProcId = getStart() + (wordIndex &lt;&lt; ADDRESS_BITS_PER_WORD);<a name="line.212"></a>
-<span class="sourceLineNo">213</span>      for (int i = 0; i &lt; (1 &lt;&lt; ADDRESS_BITS_PER_WORD); i++) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        long mask = 1L &lt;&lt; i;<a name="line.214"></a>
-<span class="sourceLineNo">215</span>        if ((deleted[wordIndex] &amp; mask) == 0 &amp;&amp; (modified[wordIndex] &amp; mask) != 0) {<a name="line.215"></a>
-<span class="sourceLineNo">216</span>          procIds.add(baseProcId + i);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        }<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      }<a name="line.218"></a>
-<span class="sourceLineNo">219</span>    }<a name="line.219"></a>
-<span class="sourceLineNo">220</span>    return procIds.stream().mapToLong(Long::longValue).toArray();<a name="line.220"></a>
-<span class="sourceLineNo">221</span>  }<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>  /**<a name="line.223"></a>
-<span class="sourceLineNo">224</span>   * @return true, if there are no active procedures in this BitSetNode, else false.<a name="line.224"></a>
-<span class="sourceLineNo">225</span>   */<a name="line.225"></a>
-<span class="sourceLineNo">226</span>  public boolean isEmpty() {<a name="line.226"></a>
-<span class="sourceLineNo">227</span>    // TODO: cache the value<a name="line.227"></a>
-<span class="sourceLineNo">228</span>    for (int i = 0; i &lt; deleted.length; ++i) {<a name="line.228"></a>
-<span class="sourceLineNo">229</span>      if (deleted[i] != WORD_MASK) {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>        return false;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      }<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    }<a name="line.232"></a>
-<span class="sourceLineNo">233</span>    return true;<a name="line.233"></a>
-<span class="sourceLineNo">234</span>  }<a name="line.234"></a>
-<span class="sourceLineNo">235</span><a name="line.235"></a>
-<span class="sourceLineNo">236</span>  public void resetModified() {<a name="line.236"></a>
-<span class="sourceLineNo">237</span>    Arrays.fill(modified, 0);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>  }<a name="line.238"></a>
-<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span>  public void unsetPartialFlag() {<a name="line.240"></a>
-<span class="sourceLineNo">241</span>    partial = false;<a name="line.241"></a>
-<span class="sourceLineNo">242</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.242"></a>
-<span class="sourceLineNo">243</span>      for (int j = 0; j &lt; BITS_PER_WORD; ++j) {<a name="line.243"></a>
-<span class="sourceLineNo">244</span>        if ((modified[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.244"></a>
-<span class="sourceLineNo">245</span>          deleted[i] |= (1L &lt;&lt; j);<a name="line.245"></a>
-<span class="sourceLineNo">246</span>        }<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      }<a name="line.247"></a>
-<span class="sourceLineNo">248</span>    }<a name="line.248"></a>
-<span class="sourceLineNo">249</span>  }<a name="line.249"></a>
-<span class="sourceLineNo">250</span><a name="line.250"></a>
-<span class="sourceLineNo">251</span>  /**<a name="line.251"></a>
-<span class="sourceLineNo">252</span>   * Convert to<a name="line.252"></a>
-<span class="sourceLineNo">253</span>   * org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureStoreTracker.TrackerNode<a name="line.253"></a>
-<span class="sourceLineNo">254</span>   * protobuf.<a name="line.254"></a>
-<span class="sourceLineNo">255</span>   */<a name="line.255"></a>
-<span class="sourceLineNo">256</span>  public ProcedureProtos.ProcedureStoreTracker.TrackerNode convert() {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>    ProcedureProtos.ProcedureStoreTracker.TrackerNode.Builder builder =<a name="line.257"></a>
-<span class="sourceLineNo">258</span>      ProcedureProtos.ProcedureStoreTracker.TrackerNode.newBuilder();<a name="line.258"></a>
-<span class="sourceLineNo">259</span>    builder.setStartId(start);<a name="line.259"></a>
-<span class="sourceLineNo">260</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span>      builder.addUpdated(modified[i]);<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      builder.addDeleted(deleted[i]);<a name="line.262"></a>
-<span class="sourceLineNo">263</span>    }<a name="line.263"></a>
-<span class="sourceLineNo">264</span>    return builder.build();<a name="line.264"></a>
-<span class="sourceLineNo">265</span>  }<a name="line.265"></a>
-<span class="sourceLineNo">266</span><a name="line.266"></a>
-<span class="sourceLineNo">267</span>  // ========================================================================<a name="line.267"></a>
-<span class="sourceLineNo">268</span>  // Grow/Merge Helpers<a name="line.268"></a>
+<span class="sourceLineNo">134</span>    // The resetDelete will be set to true when building cleanup tracker.<a name="line.134"></a>
+<span class="sourceLineNo">135</span>    // as we will reset deleted flags for all the unmodified bits to 1, the partial flag is useless<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    // so set it to false for not confusing the developers when debugging.<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    this.partial = resetDelete ? false : other.partial;<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    this.modified = other.modified.clone();<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    // The intention here is that, if a procedure is not modified in this tracker, then we do not<a name="line.139"></a>
+<span class="sourceLineNo">140</span>    // need to take care of it, so we will set deleted to true for these bits, i.e, if modified is<a name="line.140"></a>
+<span class="sourceLineNo">141</span>    // 0, then we set deleted to 1, otherwise keep it as is. So here, the equation is<a name="line.141"></a>
+<span class="sourceLineNo">142</span>    // deleted |= ~modified, i.e,<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    if (resetDelete) {<a name="line.143"></a>
+<span class="sourceLineNo">144</span>      this.deleted = new long[other.deleted.length];<a name="line.144"></a>
+<span class="sourceLineNo">145</span>      for (int i = 0; i &lt; this.deleted.length; ++i) {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>        this.deleted[i] |= ~(other.modified[i]);<a name="line.146"></a>
+<span class="sourceLineNo">147</span>      }<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    } else {<a name="line.148"></a>
+<span class="sourceLineNo">149</span>      this.deleted = other.deleted.clone();<a name="line.149"></a>
+<span class="sourceLineNo">150</span>    }<a name="line.150"></a>
+<span class="sourceLineNo">151</span>  }<a name="line.151"></a>
+<span class="sourceLineNo">152</span><a name="line.152"></a>
+<span class="sourceLineNo">153</span>  public void insertOrUpdate(final long procId) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    updateState(procId, false);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>  }<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span>  public void delete(final long procId) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    updateState(procId, true);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>  }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>  public long getStart() {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    return start;<a name="line.162"></a>
+<span class="sourceLineNo">163</span>  }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span>  public long getEnd() {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>    return start + (modified.length &lt;&lt; ADDRESS_BITS_PER_WORD) - 1;<a name="line.166"></a>
+<span class="sourceLineNo">167</span>  }<a name="line.167"></a>
+<span class="sourceLineNo">168</span><a name="line.168"></a>
+<span class="sourceLineNo">169</span>  public boolean contains(final long procId) {<a name="line.169"></a>
+<span class="sourceLineNo">170</span>    return start &lt;= procId &amp;&amp; procId &lt;= getEnd();<a name="line.170"></a>
+<span class="sourceLineNo">171</span>  }<a name="line.171"></a>
+<span class="sourceLineNo">172</span><a name="line.172"></a>
+<span class="sourceLineNo">173</span>  public DeleteState isDeleted(final long procId) {<a name="line.173"></a>
+<span class="sourceLineNo">174</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.175"></a>
+<span class="sourceLineNo">176</span>    if (wordIndex &gt;= deleted.length) {<a name="line.176"></a>
+<span class="sourceLineNo">177</span>      return DeleteState.MAYBE;<a name="line.177"></a>
+<span class="sourceLineNo">178</span>    }<a name="line.178"></a>
+<span class="sourceLineNo">179</span>    return (deleted[wordIndex] &amp; (1L &lt;&lt; bitmapIndex)) != 0 ? DeleteState.YES : DeleteState.NO;<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  }<a name="line.180"></a>
+<span class="sourceLineNo">181</span><a name="line.181"></a>
+<span class="sourceLineNo">182</span>  public boolean isModified(long procId) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.184"></a>
+<span class="sourceLineNo">185</span>    if (wordIndex &gt;= modified.length) {<a name="line.185"></a>
+<span class="sourceLineNo">186</span>      return false;<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    }<a name="line.187"></a>
+<span class="sourceLineNo">188</span>    return (modified[wordIndex] &amp; (1L &lt;&lt; bitmapIndex)) != 0;<a name="line.188"></a>
+<span class="sourceLineNo">189</span>  }<a name="line.189"></a>
+<span class="sourceLineNo">190</span><a name="line.190"></a>
+<span class="sourceLineNo">191</span>  /**<a name="line.191"></a>
+<span class="sourceLineNo">192</span>   * @return true, if all the procedures has been modified.<a name="line.192"></a>
+<span class="sourceLineNo">193</span>   */<a name="line.193"></a>
+<span class="sourceLineNo">194</span>  public boolean isAllModified() {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    // TODO: cache the value<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.196"></a>
+<span class="sourceLineNo">197</span>      if ((modified[i] | deleted[i]) != WORD_MASK) {<a name="line.197"></a>
+<span class="sourceLineNo">198</span>        return false;<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      }<a name="line.199"></a>
+<span class="sourceLineNo">200</span>    }<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    return true;<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  /**<a name="line.204"></a>
+<span class="sourceLineNo">205</span>   * @return all the active procedure ids in this bit set.<a name="line.205"></a>
+<span class="sourceLineNo">206</span>   */<a name="line.206"></a>
+<span class="sourceLineNo">207</span>  public long[] getActiveProcIds() {<a name="line.207"></a>
+<span class="sourceLineNo">208</span>    List&lt;Long&gt; procIds = new ArrayList&lt;&gt;();<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    for (int wordIndex = 0; wordIndex &lt; modified.length; wordIndex++) {<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      if (deleted[wordIndex] == WORD_MASK || modified[wordIndex] == 0) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>        // This should be the common case, where most procedures has been deleted.<a name="line.211"></a>
+<span class="sourceLineNo">212</span>        continue;<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      }<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      long baseProcId = getStart() + (wordIndex &lt;&lt; ADDRESS_BITS_PER_WORD);<a name="line.214"></a>
+<span class="sourceLineNo">215</span>      for (int i = 0; i &lt; (1 &lt;&lt; ADDRESS_BITS_PER_WORD); i++) {<a name="line.215"></a>
+<span class="sourceLineNo">216</span>        long mask = 1L &lt;&lt; i;<a name="line.216"></a>
+<span class="sourceLineNo">217</span>        if ((deleted[wordIndex] &amp; mask) == 0 &amp;&amp; (modified[wordIndex] &amp; mask) != 0) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>          procIds.add(baseProcId + i);<a name="line.218"></a>
+<span class="sourceLineNo">219</span>        }<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    }<a name="line.221"></a>
+<span class="sourceLineNo">222</span>    return procIds.stream().mapToLong(Long::longValue).toArray();<a name="line.222"></a>
+<span class="sourceLineNo">223</span>  }<a name="line.223"></a>
+<span class="sourceLineNo">224</span><a name="line.224"></a>
+<span class="sourceLineNo">225</span>  /**<a name="line.225"></a>
+<span class="sourceLineNo">226</span>   * @return true, if there are no active procedures in this BitSetNode, else false.<a name="line.226"></a>
+<span class="sourceLineNo">227</span>   */<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  public boolean isEmpty() {<a name="line.228"></a>
+<span class="sourceLineNo">229</span>    // TODO: cache the value<a name="line.229"></a>
+<span class="sourceLineNo">230</span>    for (int i = 0; i &lt; deleted.length; ++i) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span>      if (deleted[i] != WORD_MASK) {<a name="line.231"></a>
+<span class="sourceLineNo">232</span>        return false;<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      }<a name="line.233"></a>
+<span class="sourceLineNo">234</span>    }<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    return true;<a name="line.235"></a>
+<span class="sourceLineNo">236</span>  }<a name="line.236"></a>
+<span class="sourceLineNo">237</span><a name="line.237"></a>
+<span class="sourceLineNo">238</span>  public void resetModified() {<a name="line.238"></a>
+<span class="sourceLineNo">239</span>    Arrays.fill(modified, 0);<a name="line.239"></a>
+<span class="sourceLineNo">240</span>  }<a name="line.240"></a>
+<span class="sourceLineNo">241</span><a name="line.241"></a>
+<span class="sourceLineNo">242</span>  public void unsetPartialFlag() {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>    partial = false;<a name="line.243"></a>
+<span class="sourceLineNo">244</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.244"></a>
+<span class="sourceLineNo">245</span>      for (int j = 0; j &lt; BITS_PER_WORD; ++j) {<a name="line.245"></a>
+<span class="sourceLineNo">246</span>        if ((modified[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span>          deleted[i] |= (1L &lt;&lt; j);<a name="line.247"></a>
+<span class="sourceLineNo">248</span>        }<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      }<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    }<a name="line.250"></a>
+<span class="sourceLineNo">251</span>  }<a name="line.251"></a>
+<span class="sourceLineNo">252</span><a name="line.252"></a>
+<span class="sourceLineNo">253</span>  /**<a name="line.253"></a>
+<span class="sourceLineNo">254</span>   * Convert to<a name="line.254"></a>
+<span class="sourceLineNo">255</span>   * org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureStoreTracker.TrackerNode<a name="line.255"></a>
+<span class="sourceLineNo">256</span>   * protobuf.<a name="line.256"></a>
+<span class="sourceLineNo">257</span>   */<a name="line.257"></a>
+<span class="sourceLineNo">258</span>  public ProcedureProtos.ProcedureStoreTracker.TrackerNode convert() {<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    ProcedureProtos.ProcedureStoreTracker.TrackerNode.Builder builder =<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      ProcedureProtos.ProcedureStoreTracker.TrackerNode.newBuilder();<a name="line.260"></a>
+<span class="sourceLineNo">261</span>    builder.setStartId(start);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    for (int i = 0; i &lt; modified.length; ++i) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span>      builder.addUpdated(modified[i]);<a name="line.263"></a>
+<span class="sourceLineNo">264</span>      builder.addDeleted(deleted[i]);<a name="line.264"></a>
+<span class="sourceLineNo">265</span>    }<a name="line.265"></a>
+<span class="sourceLineNo">266</span>    return builder.build();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>  }<a name="line.267"></a>
+<span class="sourceLineNo">268</span><a name="line.268"></a>
 <span class="sourceLineNo">269</span>  // ========================================================================<a name="line.269"></a>
-<span class="sourceLineNo">270</span>  public boolean canGrow(final long procId) {<a name="line.270"></a>
-<span class="sourceLineNo">271</span>    return Math.abs(procId - start) &lt; MAX_NODE_SIZE;<a name="line.271"></a>
-<span class="sourceLineNo">272</span>  }<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span>  public boolean canMerge(final BitSetNode rightNode) {<a name="line.274"></a>
-<span class="sourceLineNo">275</span>    // Can just compare 'starts' since boundaries are aligned to multiples of BITS_PER_WORD.<a name="line.275"></a>
-<span class="sourceLineNo">276</span>    assert start &lt; rightNode.start;<a name="line.276"></a>
-<span class="sourceLineNo">277</span>    return (rightNode.getEnd() - start) &lt; MAX_NODE_SIZE;<a name="line.277"></a>
-<span class="sourceLineNo">278</span>  }<a name="line.278"></a>
-<span class="sourceLineNo">279</span><a name="line.279"></a>
-<span class="sourceLineNo">280</span>  public void grow(final long procId) {<a name="line.280"></a>
-<span class="sourceLineNo">281</span>    int delta, offset;<a name="line.281"></a>
-<span class="sourceLineNo">282</span><a name="line.282"></a>
-<span class="sourceLineNo">283</span>    if (procId &lt; start) {<a name="line.283"></a>
-<span class="sourceLineNo">284</span>      // add to head<a name="line.284"></a>
-<span class="sourceLineNo">285</span>      long newStart = alignDown(procId);<a name="line.285"></a>
-<span class="sourceLineNo">286</span>      delta = (int) (start - newStart) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.286"></a>
-<span class="sourceLineNo">287</span>      offset = delta;<a name="line.287"></a>
-<span class="sourceLineNo">288</span>      start = newStart;<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    } else {<a name="line.289"></a>
-<span class="sourceLineNo">290</span>      // Add to tail<a name="line.290"></a>
-<span class="sourceLineNo">291</span>      long newEnd = alignUp(procId + 1);<a name="line.291"></a>
-<span class="sourceLineNo">292</span>      delta = (int) (newEnd - getEnd()) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.292"></a>
-<span class="sourceLineNo">293</span>      offset = 0;<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    }<a name="line.294"></a>
-<span class="sourceLineNo">295</span><a name="line.295"></a>
-<span class="sourceLineNo">296</span>    long[] newBitmap;<a name="line.296"></a>
-<span class="sourceLineNo">297</span>    int oldSize = modified.length;<a name="line.297"></a>
-<span class="sourceLineNo">298</span><a name="line.298"></a>
-<span class="sourceLineNo">299</span>    newBitmap = new long[oldSize + delta];<a name="line.299"></a>
-<span class="sourceLineNo">300</span>    for (int i = 0; i &lt; newBitmap.length; ++i) {<a name="line.300"></a>
-<span class="sourceLineNo">301</span>      newBitmap[i] = 0;<a name="line.301"></a>
-<span class="sourceLineNo">302</span>    }<a name="line.302"></a>
-<span class="sourceLineNo">303</span>    System.arraycopy(modified, 0, newBitmap, offset, oldSize);<a name="line.303"></a>
-<span class="sourceLineNo">304</span>    modified = newBitmap;<a name="line.304"></a>
-<span class="sourceLineNo">305</span><a name="line.305"></a>
-<span class="sourceLineNo">306</span>    newBitmap = new long[deleted.length + delta];<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    for (int i = 0; i &lt; newBitmap.length; ++i) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span>      newBitmap[i] = partial ? 0 : WORD_MASK;<a name="line.308"></a>
-<span class="sourceLineNo">309</span>    }<a name="line.309"></a>
-<span class="sourceLineNo">310</span>    System.arraycopy(deleted, 0, newBitmap, offset, oldSize);<a name="line.310"></a>
-<span class="sourceLineNo">311</span>    deleted = newBitmap;<a name="line.311"></a>
-<span class="sourceLineNo">312</span>  }<a name="line.312"></a>
-<span class="sourceLineNo">313</span><a name="line.313"></a>
-<span class="sourceLineNo">314</span>  public void merge(final BitSetNode rightNode) {<a name="line.314"></a>
-<span class="sourceLineNo">315</span>    int delta = (int) (rightNode.getEnd() - getEnd()) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.315"></a>
-<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span>    long[] newBitmap;<a name="line.317"></a>
-<span class="sourceLineNo">318</span>    int oldSize = modified.length;<a name="line.318"></a>
-<span class="sourceLineNo">319</span>    int newSize = (delta - rightNode.modified.length);<a name="line.319"></a>
-<span class="sourceLineNo">320</span>    int offset = oldSize + newSize;<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span>    newBitmap = new long[oldSize + delta];<a name="line.322"></a>
-<span class="sourceLineNo">323</span>    System.arraycopy(modified, 0, newBitmap, 0, oldSize);<a name="line.323"></a>
-<span class="sourceLineNo">324</span>    System.arraycopy(rightNode.modified, 0, newBitmap, offset, rightNode.modified.length);<a name="line.324"></a>
-<span class="sourceLineNo">325</span>    modified = newBitmap;<a name="line.325"></a>
-<span class="sourceLineNo">326</span><a name="line.326"></a>
-<span class="sourceLineNo">327</span>    newBitmap = new long[oldSize + delta];<a name="line.327"></a>
-<span class="sourceLineNo">328</span>    System.arraycopy(deleted, 0, newBitmap, 0, oldSize);<a name="line.328"></a>
-<span class="sourceLineNo">329</span>    System.arraycopy(rightNode.deleted, 0, newBitmap, offset, rightNode.deleted.length);<a name="line.329"></a>
-<span class="sourceLineNo">330</span>    deleted = newBitmap;<a name="line.330"></a>
-<span class="sourceLineNo">331</span><a name="line.331"></a>
-<span class="sourceLineNo">332</span>    for (int i = 0; i &lt; newSize; ++i) {<a name="line.332"></a>
-<span class="sourceLineNo">333</span>      modified[offset + i] = 0;<a name="line.333"></a>
-<span class="sourceLineNo">334</span>      deleted[offset + i] = partial ? 0 : WORD_MASK;<a name="line.334"></a>
-<span class="sourceLineNo">335</span>    }<a name="line.335"></a>
-<span class="sourceLineNo">336</span>  }<a name="line.336"></a>
-<span class="sourceLineNo">337</span><a name="line.337"></a>
-<span class="sourceLineNo">338</span>  @Override<a name="line.338"></a>
-<span class="sourceLineNo">339</span>  public String toString() {<a name="line.339"></a>
-<span class="sourceLineNo">340</span>    return "BitSetNode(" + getStart() + "-" + getEnd() + ")";<a name="line.340"></a>
-<span class="sourceLineNo">341</span>  }<a name="line.341"></a>
-<span class="sourceLineNo">342</span><a name="line.342"></a>
-<span class="sourceLineNo">343</span>  // ========================================================================<a name="line.343"></a>
-<span class="sourceLineNo">344</span>  // Min/Max Helpers<a name="line.344"></a>
+<span class="sourceLineNo">270</span>  // Grow/Merge Helpers<a name="line.270"></a>
+<span class="sourceLineNo">271</span>  // ========================================================================<a name="line.271"></a>
+<span class="sourceLineNo">272</span>  public boolean canGrow(final long procId) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    return Math.abs(procId - start) &lt; MAX_NODE_SIZE;<a name="line.273"></a>
+<span class="sourceLineNo">274</span>  }<a name="line.274"></a>
+<span class="sourceLineNo">275</span><a name="line.275"></a>
+<span class="sourceLineNo">276</span>  public boolean canMerge(final BitSetNode rightNode) {<a name="line.276"></a>
+<span class="sourceLineNo">277</span>    // Can just compare 'starts' since boundaries are aligned to multiples of BITS_PER_WORD.<a name="line.277"></a>
+<span class="sourceLineNo">278</span>    assert start &lt; rightNode.start;<a name="line.278"></a>
+<span class="sourceLineNo">279</span>    return (rightNode.getEnd() - start) &lt; MAX_NODE_SIZE;<a name="line.279"></a>
+<span class="sourceLineNo">280</span>  }<a name="line.280"></a>
+<span class="sourceLineNo">281</span><a name="line.281"></a>
+<span class="sourceLineNo">282</span>  public void grow(final long procId) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span>    int delta, offset;<a name="line.283"></a>
+<span class="sourceLineNo">284</span><a name="line.284"></a>
+<span class="sourceLineNo">285</span>    if (procId &lt; start) {<a name="line.285"></a>
+<span class="sourceLineNo">286</span>      // add to head<a name="line.286"></a>
+<span class="sourceLineNo">287</span>      long newStart = alignDown(procId);<a name="line.287"></a>
+<span class="sourceLineNo">288</span>      delta = (int) (start - newStart) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.288"></a>
+<span class="sourceLineNo">289</span>      offset = delta;<a name="line.289"></a>
+<span class="sourceLineNo">290</span>      start = newStart;<a name="line.290"></a>
+<span class="sourceLineNo">291</span>    } else {<a name="line.291"></a>
+<span class="sourceLineNo">292</span>      // Add to tail<a name="line.292"></a>
+<span class="sourceLineNo">293</span>      long newEnd = alignUp(procId + 1);<a name="line.293"></a>
+<span class="sourceLineNo">294</span>      delta = (int) (newEnd - getEnd()) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.294"></a>
+<span class="sourceLineNo">295</span>      offset = 0;<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    }<a name="line.296"></a>
+<span class="sourceLineNo">297</span><a name="line.297"></a>
+<span class="sourceLineNo">298</span>    long[] newBitmap;<a name="line.298"></a>
+<span class="sourceLineNo">299</span>    int oldSize = modified.length;<a name="line.299"></a>
+<span class="sourceLineNo">300</span><a name="line.300"></a>
+<span class="sourceLineNo">301</span>    newBitmap = new long[oldSize + delta];<a name="line.301"></a>
+<span class="sourceLineNo">302</span>    for (int i = 0; i &lt; newBitmap.length; ++i) {<a name="line.302"></a>
+<span class="sourceLineNo">303</span>      newBitmap[i] = 0;<a name="line.303"></a>
+<span class="sourceLineNo">304</span>    }<a name="line.304"></a>
+<span class="sourceLineNo">305</span>    System.arraycopy(modified, 0, newBitmap, offset, oldSize);<a name="line.305"></a>
+<span class="sourceLineNo">306</span>    modified = newBitmap;<a name="line.306"></a>
+<span class="sourceLineNo">307</span><a name="line.307"></a>
+<span class="sourceLineNo">308</span>    newBitmap = new long[deleted.length + delta];<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    for (int i = 0; i &lt; newBitmap.length; ++i) {<a name="line.309"></a>
+<span class="sourceLineNo">310</span>      newBitmap[i] = partial ? 0 : WORD_MASK;<a name="line.310"></a>
+<span class="sourceLineNo">311</span>    }<a name="line.311"></a>
+<span class="sourceLineNo">312</span>    System.arraycopy(deleted, 0, newBitmap, offset, oldSize);<a name="line.312"></a>
+<span class="sourceLineNo">313</span>    deleted = newBitmap;<a name="line.313"></a>
+<span class="sourceLineNo">314</span>  }<a name="line.314"></a>
+<span class="sourceLineNo">315</span><a name="line.315"></a>
+<span class="sourceLineNo">316</span>  public void merge(final BitSetNode rightNode) {<a name="line.316"></a>
+<span class="sourceLineNo">317</span>    int delta = (int) (rightNode.getEnd() - getEnd()) &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.317"></a>
+<span class="sourceLineNo">318</span><a name="line.318"></a>
+<span class="sourceLineNo">319</span>    long[] newBitmap;<a name="line.319"></a>
+<span class="sourceLineNo">320</span>    int oldSize = modified.length;<a name="line.320"></a>
+<span class="sourceLineNo">321</span>    int newSize = (delta - rightNode.modified.length);<a name="line.321"></a>
+<span class="sourceLineNo">322</span>    int offset = oldSize + newSize;<a name="line.322"></a>
+<span class="sourceLineNo">323</span><a name="line.323"></a>
+<span class="sourceLineNo">324</span>    newBitmap = new long[oldSize + delta];<a name="line.324"></a>
+<span class="sourceLineNo">325</span>    System.arraycopy(modified, 0, newBitmap, 0, oldSize);<a name="line.325"></a>
+<span class="sourceLineNo">326</span>    System.arraycopy(rightNode.modified, 0, newBitmap, offset, rightNode.modified.length);<a name="line.326"></a>
+<span class="sourceLineNo">327</span>    modified = newBitmap;<a name="line.327"></a>
+<span class="sourceLineNo">328</span><a name="line.328"></a>
+<span class="sourceLineNo">329</span>    newBitmap = new long[oldSize + delta];<a name="line.329"></a>
+<span class="sourceLineNo">330</span>    System.arraycopy(deleted, 0, newBitmap, 0, oldSize);<a name="line.330"></a>
+<span class="sourceLineNo">331</span>    System.arraycopy(rightNode.deleted, 0, newBitmap, offset, rightNode.deleted.length);<a name="line.331"></a>
+<span class="sourceLineNo">332</span>    deleted = newBitmap;<a name="line.332"></a>
+<span class="sourceLineNo">333</span><a name="line.333"></a>
+<span class="sourceLineNo">334</span>    for (int i = 0; i &lt; newSize; ++i) {<a name="line.334"></a>
+<span class="sourceLineNo">335</span>      modified[offset + i] = 0;<a name="line.335"></a>
+<span class="sourceLineNo">336</span>      deleted[offset + i] = partial ? 0 : WORD_MASK;<a name="line.336"></a>
+<span class="sourceLineNo">337</span>    }<a name="line.337"></a>
+<span class="sourceLineNo">338</span>  }<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span>  @Override<a name="line.340"></a>
+<span class="sourceLineNo">341</span>  public String toString() {<a name="line.341"></a>
+<span class="sourceLineNo">342</span>    return "BitSetNode(" + getStart() + "-" + getEnd() + ")";<a name="line.342"></a>
+<span class="sourceLineNo">343</span>  }<a name="line.343"></a>
+<span class="sourceLineNo">344</span><a name="line.344"></a>
 <span class="sourceLineNo">345</span>  // ========================================================================<a name="line.345"></a>
-<span class="sourceLineNo">346</span>  public long getActiveMinProcId() {<a name="line.346"></a>
-<span class="sourceLineNo">347</span>    long minProcId = start;<a name="line.347"></a>
-<span class="sourceLineNo">348</span>    for (int i = 0; i &lt; deleted.length; ++i) {<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      if (deleted[i] == 0) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span>        return minProcId;<a name="line.350"></a>
-<span class="sourceLineNo">351</span>      }<a name="line.351"></a>
-<span class="sourceLineNo">352</span><a name="line.352"></a>
-<span class="sourceLineNo">353</span>      if (deleted[i] != WORD_MASK) {<a name="line.353"></a>
-<span class="sourceLineNo">354</span>        for (int j = 0; j &lt; BITS_PER_WORD; ++j) {<a name="line.354"></a>
-<span class="sourceLineNo">355</span>          if ((deleted[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.355"></a>
-<span class="sourceLineNo">356</span>            return minProcId + j;<a name="line.356"></a>
-<span class="sourceLineNo">357</span>          }<a name="line.357"></a>
-<span class="sourceLineNo">358</span>        }<a name="line.358"></a>
-<span class="sourceLineNo">359</span>      }<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span>      minProcId += BITS_PER_WORD;<a name="line.361"></a>
-<span class="sourceLineNo">362</span>    }<a name="line.362"></a>
-<span class="sourceLineNo">363</span>    return Procedure.NO_PROC_ID;<a name="line.363"></a>
-<span class="sourceLineNo">364</span>  }<a name="line.364"></a>
-<span class="sourceLineNo">365</span><a name="line.365"></a>
-<span class="sourceLineNo">366</span>  public long getActiveMaxProcId() {<a name="line.366"></a>
-<span class="sourceLineNo">367</span>    long maxProcId = getEnd();<a name="line.367"></a>
-<span class="sourceLineNo">368</span>    for (int i = deleted.length - 1; i &gt;= 0; --i) {<a name="line.368"></a>
-<span class="sourceLineNo">369</span>      if (deleted[i] == 0) {<a name="line.369"></a>
-<span class="sourceLineNo">370</span>        return maxProcId;<a name="line.370"></a>
-<span class="sourceLineNo">371</span>      }<a name="line.371"></a>
-<span class="sourceLineNo">372</span><a name="line.372"></a>
-<span class="sourceLineNo">373</span>      if (deleted[i] != WORD_MASK) {<a name="line.373"></a>
-<span class="sourceLineNo">374</span>        for (int j = BITS_PER_WORD - 1; j &gt;= 0; --j) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span>          if ((deleted[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.375"></a>
-<span class="sourceLineNo">376</span>            return maxProcId - (BITS_PER_WORD - 1 - j);<a name="line.376"></a>
-<span class="sourceLineNo">377</span>          }<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        }<a name="line.378"></a>
-<span class="sourceLineNo">379</span>      }<a name="line.379"></a>
-<span class="sourceLineNo">380</span>      maxProcId -= BITS_PER_WORD;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    }<a name="line.381"></a>
-<span class="sourceLineNo">382</span>    return Procedure.NO_PROC_ID;<a name="line.382"></a>
-<span class="sourceLineNo">383</span>  }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span>  // ========================================================================<a name="line.385"></a>
-<span class="sourceLineNo">386</span>  // Bitmap Helpers<a name="line.386"></a>
+<span class="sourceLineNo">346</span>  // Min/Max Helpers<a name="line.346"></a>
+<span class="sourceLineNo">347</span>  // ========================================================================<a name="line.347"></a>
+<span class="sourceLineNo">348</span>  public long getActiveMinProcId() {<a name="line.348"></a>
+<span class="sourceLineNo">349</span>    long minProcId = start;<a name="line.349"></a>
+<span class="sourceLineNo">350</span>    for (int i = 0; i &lt; deleted.length; ++i) {<a name="line.350"></a>
+<span class="sourceLineNo">351</span>      if (deleted[i] == 0) {<a name="line.351"></a>
+<span class="sourceLineNo">352</span>        return minProcId;<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      }<a name="line.353"></a>
+<span class="sourceLineNo">354</span><a name="line.354"></a>
+<span class="sourceLineNo">355</span>      if (deleted[i] != WORD_MASK) {<a name="line.355"></a>
+<span class="sourceLineNo">356</span>        for (int j = 0; j &lt; BITS_PER_WORD; ++j) {<a name="line.356"></a>
+<span class="sourceLineNo">357</span>          if ((deleted[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.357"></a>
+<span class="sourceLineNo">358</span>            return minProcId + j;<a name="line.358"></a>
+<span class="sourceLineNo">359</span>          }<a name="line.359"></a>
+<span class="sourceLineNo">360</span>        }<a name="line.360"></a>
+<span class="sourceLineNo">361</span>      }<a name="line.361"></a>
+<span class="sourceLineNo">362</span><a name="line.362"></a>
+<span class="sourceLineNo">363</span>      minProcId += BITS_PER_WORD;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
+<span class="sourceLineNo">365</span>    return Procedure.NO_PROC_ID;<a name="line.365"></a>
+<span class="sourceLineNo">366</span>  }<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span>  public long getActiveMaxProcId() {<a name="line.368"></a>
+<span class="sourceLineNo">369</span>    long maxProcId = getEnd();<a name="line.369"></a>
+<span class="sourceLineNo">370</span>    for (int i = deleted.length - 1; i &gt;= 0; --i) {<a name="line.370"></a>
+<span class="sourceLineNo">371</span>      if (deleted[i] == 0) {<a name="line.371"></a>
+<span class="sourceLineNo">372</span>        return maxProcId;<a name="line.372"></a>
+<span class="sourceLineNo">373</span>      }<a name="line.373"></a>
+<span class="sourceLineNo">374</span><a name="line.374"></a>
+<span class="sourceLineNo">375</span>      if (deleted[i] != WORD_MASK) {<a name="line.375"></a>
+<span class="sourceLineNo">376</span>        for (int j = BITS_PER_WORD - 1; j &gt;= 0; --j) {<a name="line.376"></a>
+<span class="sourceLineNo">377</span>          if ((deleted[i] &amp; (1L &lt;&lt; j)) == 0) {<a name="line.377"></a>
+<span class="sourceLineNo">378</span>            return maxProcId - (BITS_PER_WORD - 1 - j);<a name="line.378"></a>
+<span class="sourceLineNo">379</span>          }<a name="line.379"></a>
+<span class="sourceLineNo">380</span>        }<a name="line.380"></a>
+<span class="sourceLineNo">381</span>      }<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      maxProcId -= BITS_PER_WORD;<a name="line.382"></a>
+<span class="sourceLineNo">383</span>    }<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    return Procedure.NO_PROC_ID;<a name="line.384"></a>
+<span class="sourceLineNo">385</span>  }<a name="line.385"></a>
+<span class="sourceLineNo">386</span><a name="line.386"></a>
 <span class="sourceLineNo">387</span>  // ========================================================================<a name="line.387"></a>
-<span class="sourceLineNo">388</span>  private int getBitmapIndex(final long procId) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>    return (int) (procId - start);<a name="line.389"></a>
-<span class="sourceLineNo">390</span>  }<a name="line.390"></a>
-<span class="sourceLineNo">391</span><a name="line.391"></a>
-<span class="sourceLineNo">392</span>  void updateState(long procId, boolean isDeleted) {<a name="line.392"></a>
-<span class="sourceLineNo">393</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.393"></a>
-<span class="sourceLineNo">394</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.394"></a>
-<span class="sourceLineNo">395</span>    long value = (1L &lt;&lt; bitmapIndex);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>    modified[wordIndex] |= value;<a name="line.397"></a>
-<span class="sourceLineNo">398</span>    if (isDeleted) {<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      deleted[wordIndex] |= value;<a name="line.399"></a>
-<span class="sourceLineNo">400</span>    } else {<a name="line.400"></a>
-<span class="sourceLineNo">401</span>      deleted[wordIndex] &amp;= ~value;<a name="line.401"></a>
-<span class="sourceLineNo">402</span>    }<a name="line.402"></a>
-<span class="sourceLineNo">403</span>  }<a name="line.403"></a>
-<span class="sourceLineNo">404</span><a name="line.404"></a>
-<span class="sourceLineNo">405</span>  // ========================================================================<a name="line.405"></a>
-<span class="sourceLineNo">406</span>  // Helpers<a name="line.406"></a>
+<span class="sourceLineNo">388</span>  // Bitmap Helpers<a name="line.388"></a>
+<span class="sourceLineNo">389</span>  // ========================================================================<a name="line.389"></a>
+<span class="sourceLineNo">390</span>  private int getBitmapIndex(final long procId) {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>    return (int) (procId - start);<a name="line.391"></a>
+<span class="sourceLineNo">392</span>  }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span>  void updateState(long procId, boolean isDeleted) {<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    int bitmapIndex = getBitmapIndex(procId);<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    int wordIndex = bitmapIndex &gt;&gt; ADDRESS_BITS_PER_WORD;<a name="line.396"></a>
+<span class="sourceLineNo">397</span>    long value = (1L &lt;&lt; bitmapIndex);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    modified[wordIndex] |= value;<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    if (isDeleted) {<a name="line.400"></a>
+<span class="sourceLineNo">401</span>      deleted[wordIndex] |= value;<a name="line.401"></a>
+<span class="sourceLineNo">402</span>    } else {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      deleted[wordIndex] &amp;= ~value;<a name="line.403"></a>
+<span class="sourceLineNo">404</span>    }<a name="line.404"></a>
+<span class="sourceLineNo">405</span>  }<a name="line.405"></a>
+<span class="sourceLineNo">406</span><a name="line.406"></a>
 <span class="sourceLineNo">407</span>  // ========================================================================<a name="line.407"></a>
-<span class="sourceLineNo">408</span>  /**<a name="line.408"></a>
-<span class="sourceLineNo">409</span>   * @return upper boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.<a name="line.409"></a>
-<span class="sourceLineNo">410</span>   */<a name="line.410"></a>
-<span class="sourceLineNo">411</span>  private static long alignUp(final long x) {<a name="line.411"></a>
-<span class="sourceLineNo">412</span>    return (x + (BITS_PER_WORD - 1)) &amp; -BITS_PER_WORD;<a name="line.412"></a>
-<span class="sourceLineNo">413</span>  }<a name="line.413"></a>
-<span class="sourceLineNo">414</span><a name="line.414"></a>
-<span class="sourceLineNo">415</span>  /**<a name="line.415"></a>
-<span class="sourceLineNo">416</span>   * @return lower boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>   */<a name="line.417"></a>
-<span class="sourceLineNo">418</span>  private static long alignDown(final long x) {<a name="line.418"></a>
-<span class="sourceLineNo">419</span>    return x &amp; -BITS_PER_WORD;<a name="line.419"></a>
-<span class="sourceLineNo">420</span>  }<a name="line.420"></a>
-<span class="sourceLineNo">421</span>}<a name="line.421"></a>
+<span class="sourceLineNo">408</span>  // Helpers<a name="line.408"></a>
+<span class="sourceLineNo">409</span>  // ========================================================================<a name="line.409"></a>
+<span class="sourceLineNo">410</span>  /**<a name="line.410"></a>
+<span class="sourceLineNo">411</span>   * @return upper boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.<a name="line.411"></a>
+<span class="sourceLineNo">412</span>   */<a name="line.412"></a>
+<span class="sourceLineNo">413</span>  private static long alignUp(final long x) {<a name="line.413"></a>
+<span class="sourceLineNo">414</span>    return (x + (BITS_PER_WORD - 1)) &amp; -BITS_PER_WORD;<a name="line.414"></a>
+<span class="sourceLineNo">415</span>  }<a name="line.415"></a>
+<span class="sourceLineNo">416</span><a name="line.416"></a>
+<span class="sourceLineNo">417</span>  /**<a name="line.417"></a>
+<span class="sourceLineNo">418</span>   * @return lower boundary (aligned to multiple of BITS_PER_WORD) of bitmap range x belongs to.<a name="line.418"></a>
+<span class="sourceLineNo">419</span>   */<a name="line.419"></a>
+<span class="sourceLineNo">420</span>  private static long alignDown(final long x) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>    return x &amp; -BITS_PER_WORD;<a name="line.421"></a>
+<span class="sourceLineNo">422</span>  }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>}<a name="line.423"></a>
 
 
 


[26/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.PrimaryRegionCountSkewCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLo

<TRUNCATED>

[10/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
index 1579eab..dc5908e 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.DeleteState.html
@@ -31,221 +31,221 @@
 <span class="sourceLineNo">023</span>import java.util.Iterator;<a name="line.23"></a>
 <span class="sourceLineNo">024</span>import java.util.Map;<a name="line.24"></a>
 <span class="sourceLineNo">025</span>import java.util.TreeMap;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import java.util.stream.LongStream;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.procedure2.Procedure;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.29"></a>
-<span class="sourceLineNo">030</span><a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;<a name="line.31"></a>
-<span class="sourceLineNo">032</span><a name="line.32"></a>
-<span class="sourceLineNo">033</span>/**<a name="line.33"></a>
-<span class="sourceLineNo">034</span> * Keeps track of live procedures.<a name="line.34"></a>
-<span class="sourceLineNo">035</span> *<a name="line.35"></a>
-<span class="sourceLineNo">036</span> * It can be used by the ProcedureStore to identify which procedures are already<a name="line.36"></a>
-<span class="sourceLineNo">037</span> * deleted/completed to avoid the deserialization step on restart<a name="line.37"></a>
-<span class="sourceLineNo">038</span> */<a name="line.38"></a>
-<span class="sourceLineNo">039</span>@InterfaceAudience.Private<a name="line.39"></a>
-<span class="sourceLineNo">040</span>@InterfaceStability.Evolving<a name="line.40"></a>
-<span class="sourceLineNo">041</span>public class ProcedureStoreTracker {<a name="line.41"></a>
-<span class="sourceLineNo">042</span>  // Key is procedure id corresponding to first bit of the bitmap.<a name="line.42"></a>
-<span class="sourceLineNo">043</span>  private final TreeMap&lt;Long, BitSetNode&gt; map = new TreeMap&lt;&gt;();<a name="line.43"></a>
-<span class="sourceLineNo">044</span><a name="line.44"></a>
-<span class="sourceLineNo">045</span>  /**<a name="line.45"></a>
-<span class="sourceLineNo">046</span>   * If true, do not remove bits corresponding to deleted procedures. Note that this can result<a name="line.46"></a>
-<span class="sourceLineNo">047</span>   * in huge bitmaps overtime.<a name="line.47"></a>
-<span class="sourceLineNo">048</span>   * Currently, it's set to true only when building tracker state from logs during recovery. During<a name="line.48"></a>
-<span class="sourceLineNo">049</span>   * recovery, if we are sure that a procedure has been deleted, reading its old update entries<a name="line.49"></a>
-<span class="sourceLineNo">050</span>   * can be skipped.<a name="line.50"></a>
-<span class="sourceLineNo">051</span>   */<a name="line.51"></a>
-<span class="sourceLineNo">052</span>  private boolean keepDeletes = false;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>  /**<a name="line.53"></a>
-<span class="sourceLineNo">054</span>   * If true, it means tracker has incomplete information about the active/deleted procedures.<a name="line.54"></a>
-<span class="sourceLineNo">055</span>   * It's set to true only when recovering from old logs. See {@link #isDeleted(long)} docs to<a name="line.55"></a>
-<span class="sourceLineNo">056</span>   * understand it's real use.<a name="line.56"></a>
-<span class="sourceLineNo">057</span>   */<a name="line.57"></a>
-<span class="sourceLineNo">058</span>  boolean partial = false;<a name="line.58"></a>
-<span class="sourceLineNo">059</span><a name="line.59"></a>
-<span class="sourceLineNo">060</span>  private long minModifiedProcId = Long.MAX_VALUE;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>  private long maxModifiedProcId = Long.MIN_VALUE;<a name="line.61"></a>
-<span class="sourceLineNo">062</span><a name="line.62"></a>
-<span class="sourceLineNo">063</span>  public enum DeleteState { YES, NO, MAYBE }<a name="line.63"></a>
-<span class="sourceLineNo">064</span><a name="line.64"></a>
-<span class="sourceLineNo">065</span>  public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) {<a name="line.65"></a>
-<span class="sourceLineNo">066</span>    reset();<a name="line.66"></a>
-<span class="sourceLineNo">067</span>    for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) {<a name="line.67"></a>
-<span class="sourceLineNo">068</span>      final BitSetNode node = new BitSetNode(protoNode);<a name="line.68"></a>
-<span class="sourceLineNo">069</span>      map.put(node.getStart(), node);<a name="line.69"></a>
-<span class="sourceLineNo">070</span>    }<a name="line.70"></a>
-<span class="sourceLineNo">071</span>  }<a name="line.71"></a>
-<span class="sourceLineNo">072</span><a name="line.72"></a>
-<span class="sourceLineNo">073</span>  /**<a name="line.73"></a>
-<span class="sourceLineNo">074</span>   * Resets internal state to same as given {@code tracker}. Does deep copy of the bitmap.<a name="line.74"></a>
-<span class="sourceLineNo">075</span>   */<a name="line.75"></a>
-<span class="sourceLineNo">076</span>  public void resetTo(ProcedureStoreTracker tracker) {<a name="line.76"></a>
-<span class="sourceLineNo">077</span>    resetTo(tracker, false);<a name="line.77"></a>
-<span class="sourceLineNo">078</span>  }<a name="line.78"></a>
-<span class="sourceLineNo">079</span><a name="line.79"></a>
-<span class="sourceLineNo">080</span>  /**<a name="line.80"></a>
-<span class="sourceLineNo">081</span>   * Resets internal state to same as given {@code tracker}, and change the deleted flag according<a name="line.81"></a>
-<span class="sourceLineNo">082</span>   * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap.<a name="line.82"></a>
-<span class="sourceLineNo">083</span>   * &lt;p/&gt;<a name="line.83"></a>
-<span class="sourceLineNo">084</span>   * The {@code resetDelete} will be set to true when building cleanup tracker, please see the<a name="line.84"></a>
-<span class="sourceLineNo">085</span>   * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the<a name="line.85"></a>
-<span class="sourceLineNo">086</span>   * deleted flag if {@code resetDelete} is true.<a name="line.86"></a>
-<span class="sourceLineNo">087</span>   */<a name="line.87"></a>
-<span class="sourceLineNo">088</span>  public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) {<a name="line.88"></a>
-<span class="sourceLineNo">089</span>    reset();<a name="line.89"></a>
-<span class="sourceLineNo">090</span>    this.partial = tracker.partial;<a name="line.90"></a>
-<span class="sourceLineNo">091</span>    this.minModifiedProcId = tracker.minModifiedProcId;<a name="line.91"></a>
-<span class="sourceLineNo">092</span>    this.maxModifiedProcId = tracker.maxModifiedProcId;<a name="line.92"></a>
-<span class="sourceLineNo">093</span>    this.keepDeletes = tracker.keepDeletes;<a name="line.93"></a>
-<span class="sourceLineNo">094</span>    for (Map.Entry&lt;Long, BitSetNode&gt; entry : tracker.map.entrySet()) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span>      map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete));<a name="line.95"></a>
-<span class="sourceLineNo">096</span>    }<a name="line.96"></a>
-<span class="sourceLineNo">097</span>  }<a name="line.97"></a>
-<span class="sourceLineNo">098</span><a name="line.98"></a>
-<span class="sourceLineNo">099</span>  public void insert(long procId) {<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    insert(null, procId);<a name="line.100"></a>
+<span class="sourceLineNo">026</span>import java.util.function.BiFunction;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import java.util.stream.LongStream;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.procedure2.Procedure;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;<a name="line.32"></a>
+<span class="sourceLineNo">033</span><a name="line.33"></a>
+<span class="sourceLineNo">034</span>/**<a name="line.34"></a>
+<span class="sourceLineNo">035</span> * Keeps track of live procedures.<a name="line.35"></a>
+<span class="sourceLineNo">036</span> *<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * It can be used by the ProcedureStore to identify which procedures are already<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * deleted/completed to avoid the deserialization step on restart<a name="line.38"></a>
+<span class="sourceLineNo">039</span> */<a name="line.39"></a>
+<span class="sourceLineNo">040</span>@InterfaceAudience.Private<a name="line.40"></a>
+<span class="sourceLineNo">041</span>@InterfaceStability.Evolving<a name="line.41"></a>
+<span class="sourceLineNo">042</span>public class ProcedureStoreTracker {<a name="line.42"></a>
+<span class="sourceLineNo">043</span>  // Key is procedure id corresponding to first bit of the bitmap.<a name="line.43"></a>
+<span class="sourceLineNo">044</span>  private final TreeMap&lt;Long, BitSetNode&gt; map = new TreeMap&lt;&gt;();<a name="line.44"></a>
+<span class="sourceLineNo">045</span><a name="line.45"></a>
+<span class="sourceLineNo">046</span>  /**<a name="line.46"></a>
+<span class="sourceLineNo">047</span>   * If true, do not remove bits corresponding to deleted procedures. Note that this can result<a name="line.47"></a>
+<span class="sourceLineNo">048</span>   * in huge bitmaps overtime.<a name="line.48"></a>
+<span class="sourceLineNo">049</span>   * Currently, it's set to true only when building tracker state from logs during recovery. During<a name="line.49"></a>
+<span class="sourceLineNo">050</span>   * recovery, if we are sure that a procedure has been deleted, reading its old update entries<a name="line.50"></a>
+<span class="sourceLineNo">051</span>   * can be skipped.<a name="line.51"></a>
+<span class="sourceLineNo">052</span>   */<a name="line.52"></a>
+<span class="sourceLineNo">053</span>  private boolean keepDeletes = false;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>  /**<a name="line.54"></a>
+<span class="sourceLineNo">055</span>   * If true, it means tracker has incomplete information about the active/deleted procedures.<a name="line.55"></a>
+<span class="sourceLineNo">056</span>   * It's set to true only when recovering from old logs. See {@link #isDeleted(long)} docs to<a name="line.56"></a>
+<span class="sourceLineNo">057</span>   * understand it's real use.<a name="line.57"></a>
+<span class="sourceLineNo">058</span>   */<a name="line.58"></a>
+<span class="sourceLineNo">059</span>  boolean partial = false;<a name="line.59"></a>
+<span class="sourceLineNo">060</span><a name="line.60"></a>
+<span class="sourceLineNo">061</span>  private long minModifiedProcId = Long.MAX_VALUE;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>  private long maxModifiedProcId = Long.MIN_VALUE;<a name="line.62"></a>
+<span class="sourceLineNo">063</span><a name="line.63"></a>
+<span class="sourceLineNo">064</span>  public enum DeleteState { YES, NO, MAYBE }<a name="line.64"></a>
+<span class="sourceLineNo">065</span><a name="line.65"></a>
+<span class="sourceLineNo">066</span>  public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) {<a name="line.66"></a>
+<span class="sourceLineNo">067</span>    reset();<a name="line.67"></a>
+<span class="sourceLineNo">068</span>    for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) {<a name="line.68"></a>
+<span class="sourceLineNo">069</span>      final BitSetNode node = new BitSetNode(protoNode);<a name="line.69"></a>
+<span class="sourceLineNo">070</span>      map.put(node.getStart(), node);<a name="line.70"></a>
+<span class="sourceLineNo">071</span>    }<a name="line.71"></a>
+<span class="sourceLineNo">072</span>  }<a name="line.72"></a>
+<span class="sourceLineNo">073</span><a name="line.73"></a>
+<span class="sourceLineNo">074</span>  /**<a name="line.74"></a>
+<span class="sourceLineNo">075</span>   * Resets internal state to same as given {@code tracker}. Does deep copy of the bitmap.<a name="line.75"></a>
+<span class="sourceLineNo">076</span>   */<a name="line.76"></a>
+<span class="sourceLineNo">077</span>  public void resetTo(ProcedureStoreTracker tracker) {<a name="line.77"></a>
+<span class="sourceLineNo">078</span>    resetTo(tracker, false);<a name="line.78"></a>
+<span class="sourceLineNo">079</span>  }<a name="line.79"></a>
+<span class="sourceLineNo">080</span><a name="line.80"></a>
+<span class="sourceLineNo">081</span>  /**<a name="line.81"></a>
+<span class="sourceLineNo">082</span>   * Resets internal state to same as given {@code tracker}, and change the deleted flag according<a name="line.82"></a>
+<span class="sourceLineNo">083</span>   * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap.<a name="line.83"></a>
+<span class="sourceLineNo">084</span>   * &lt;p/&gt;<a name="line.84"></a>
+<span class="sourceLineNo">085</span>   * The {@code resetDelete} will be set to true when building cleanup tracker, please see the<a name="line.85"></a>
+<span class="sourceLineNo">086</span>   * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the<a name="line.86"></a>
+<span class="sourceLineNo">087</span>   * deleted flag if {@code resetDelete} is true.<a name="line.87"></a>
+<span class="sourceLineNo">088</span>   */<a name="line.88"></a>
+<span class="sourceLineNo">089</span>  public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) {<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    reset();<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    // resetDelete will true if we are building the cleanup tracker, as we will reset deleted flags<a name="line.91"></a>
+<span class="sourceLineNo">092</span>    // for all the unmodified bits to 1, the partial flag is useless so set it to false for not<a name="line.92"></a>
+<span class="sourceLineNo">093</span>    // confusing the developers when debugging.<a name="line.93"></a>
+<span class="sourceLineNo">094</span>    this.partial = resetDelete ? false : tracker.partial;<a name="line.94"></a>
+<span class="sourceLineNo">095</span>    this.minModifiedProcId = tracker.minModifiedProcId;<a name="line.95"></a>
+<span class="sourceLineNo">096</span>    this.maxModifiedProcId = tracker.maxModifiedProcId;<a name="line.96"></a>
+<span class="sourceLineNo">097</span>    this.keepDeletes = tracker.keepDeletes;<a name="line.97"></a>
+<span class="sourceLineNo">098</span>    for (Map.Entry&lt;Long, BitSetNode&gt; entry : tracker.map.entrySet()) {<a name="line.98"></a>
+<span class="sourceLineNo">099</span>      map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete));<a name="line.99"></a>
+<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
 <span class="sourceLineNo">101</span>  }<a name="line.101"></a>
 <span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public void insert(long[] procIds) {<a name="line.103"></a>
-<span class="sourceLineNo">104</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>      insert(procIds[i]);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>    }<a name="line.106"></a>
-<span class="sourceLineNo">107</span>  }<a name="line.107"></a>
-<span class="sourceLineNo">108</span><a name="line.108"></a>
-<span class="sourceLineNo">109</span>  public void insert(long procId, long[] subProcIds) {<a name="line.109"></a>
-<span class="sourceLineNo">110</span>    BitSetNode node = update(null, procId);<a name="line.110"></a>
-<span class="sourceLineNo">111</span>    for (int i = 0; i &lt; subProcIds.length; ++i) {<a name="line.111"></a>
-<span class="sourceLineNo">112</span>      node = insert(node, subProcIds[i]);<a name="line.112"></a>
-<span class="sourceLineNo">113</span>    }<a name="line.113"></a>
-<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span>  private BitSetNode insert(BitSetNode node, long procId) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>    if (node == null || !node.contains(procId)) {<a name="line.117"></a>
-<span class="sourceLineNo">118</span>      node = getOrCreateNode(procId);<a name="line.118"></a>
-<span class="sourceLineNo">119</span>    }<a name="line.119"></a>
-<span class="sourceLineNo">120</span>    node.insertOrUpdate(procId);<a name="line.120"></a>
-<span class="sourceLineNo">121</span>    trackProcIds(procId);<a name="line.121"></a>
-<span class="sourceLineNo">122</span>    return node;<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span>  public void update(long procId) {<a name="line.125"></a>
-<span class="sourceLineNo">126</span>    update(null, procId);<a name="line.126"></a>
+<span class="sourceLineNo">103</span>  public void insert(long procId) {<a name="line.103"></a>
+<span class="sourceLineNo">104</span>    insert(null, procId);<a name="line.104"></a>
+<span class="sourceLineNo">105</span>  }<a name="line.105"></a>
+<span class="sourceLineNo">106</span><a name="line.106"></a>
+<span class="sourceLineNo">107</span>  public void insert(long[] procIds) {<a name="line.107"></a>
+<span class="sourceLineNo">108</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span>      insert(procIds[i]);<a name="line.109"></a>
+<span class="sourceLineNo">110</span>    }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>  }<a name="line.111"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span>  public void insert(long procId, long[] subProcIds) {<a name="line.113"></a>
+<span class="sourceLineNo">114</span>    BitSetNode node = update(null, procId);<a name="line.114"></a>
+<span class="sourceLineNo">115</span>    for (int i = 0; i &lt; subProcIds.length; ++i) {<a name="line.115"></a>
+<span class="sourceLineNo">116</span>      node = insert(node, subProcIds[i]);<a name="line.116"></a>
+<span class="sourceLineNo">117</span>    }<a name="line.117"></a>
+<span class="sourceLineNo">118</span>  }<a name="line.118"></a>
+<span class="sourceLineNo">119</span><a name="line.119"></a>
+<span class="sourceLineNo">120</span>  private BitSetNode insert(BitSetNode node, long procId) {<a name="line.120"></a>
+<span class="sourceLineNo">121</span>    if (node == null || !node.contains(procId)) {<a name="line.121"></a>
+<span class="sourceLineNo">122</span>      node = getOrCreateNode(procId);<a name="line.122"></a>
+<span class="sourceLineNo">123</span>    }<a name="line.123"></a>
+<span class="sourceLineNo">124</span>    node.insertOrUpdate(procId);<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    trackProcIds(procId);<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    return node;<a name="line.126"></a>
 <span class="sourceLineNo">127</span>  }<a name="line.127"></a>
 <span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span>  private BitSetNode update(BitSetNode node, long procId) {<a name="line.129"></a>
-<span class="sourceLineNo">130</span>    node = lookupClosestNode(node, procId);<a name="line.130"></a>
-<span class="sourceLineNo">131</span>    assert node != null : "expected node to update procId=" + procId;<a name="line.131"></a>
-<span class="sourceLineNo">132</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    node.insertOrUpdate(procId);<a name="line.133"></a>
-<span class="sourceLineNo">134</span>    trackProcIds(procId);<a name="line.134"></a>
-<span class="sourceLineNo">135</span>    return node;<a name="line.135"></a>
-<span class="sourceLineNo">136</span>  }<a name="line.136"></a>
-<span class="sourceLineNo">137</span><a name="line.137"></a>
-<span class="sourceLineNo">138</span>  public void delete(long procId) {<a name="line.138"></a>
-<span class="sourceLineNo">139</span>    delete(null, procId);<a name="line.139"></a>
+<span class="sourceLineNo">129</span>  public void update(long procId) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span>    update(null, procId);<a name="line.130"></a>
+<span class="sourceLineNo">131</span>  }<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span>  private BitSetNode update(BitSetNode node, long procId) {<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    node = lookupClosestNode(node, procId);<a name="line.134"></a>
+<span class="sourceLineNo">135</span>    assert node != null : "expected node to update procId=" + procId;<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    node.insertOrUpdate(procId);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    trackProcIds(procId);<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return node;<a name="line.139"></a>
 <span class="sourceLineNo">140</span>  }<a name="line.140"></a>
 <span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span>  public void delete(final long[] procIds) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    Arrays.sort(procIds);<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    BitSetNode node = null;<a name="line.144"></a>
-<span class="sourceLineNo">145</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.145"></a>
-<span class="sourceLineNo">146</span>      node = delete(node, procIds[i]);<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    }<a name="line.147"></a>
-<span class="sourceLineNo">148</span>  }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>  private BitSetNode delete(BitSetNode node, long procId) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>    node = lookupClosestNode(node, procId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    assert node != null : "expected node to delete procId=" + procId;<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    node.delete(procId);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    if (!keepDeletes &amp;&amp; node.isEmpty()) {<a name="line.155"></a>
-<span class="sourceLineNo">156</span>      // TODO: RESET if (map.size() == 1)<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      map.remove(node.getStart());<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span>    trackProcIds(procId);<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    return node;<a name="line.161"></a>
-<span class="sourceLineNo">162</span>  }<a name="line.162"></a>
+<span class="sourceLineNo">142</span>  public void delete(long procId) {<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    delete(null, procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span>  }<a name="line.144"></a>
+<span class="sourceLineNo">145</span><a name="line.145"></a>
+<span class="sourceLineNo">146</span>  public void delete(final long[] procIds) {<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    Arrays.sort(procIds);<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    BitSetNode node = null;<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.149"></a>
+<span class="sourceLineNo">150</span>      node = delete(node, procIds[i]);<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    }<a name="line.151"></a>
+<span class="sourceLineNo">152</span>  }<a name="line.152"></a>
+<span class="sourceLineNo">153</span><a name="line.153"></a>
+<span class="sourceLineNo">154</span>  private BitSetNode delete(BitSetNode node, long procId) {<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    node = lookupClosestNode(node, procId);<a name="line.155"></a>
+<span class="sourceLineNo">156</span>    assert node != null : "expected node to delete procId=" + procId;<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    node.delete(procId);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    if (!keepDeletes &amp;&amp; node.isEmpty()) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>      // TODO: RESET if (map.size() == 1)<a name="line.160"></a>
+<span class="sourceLineNo">161</span>      map.remove(node.getStart());<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
 <span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span>  /**<a name="line.164"></a>
-<span class="sourceLineNo">165</span>   * Will be called when restarting where we need to rebuild the ProcedureStoreTracker.<a name="line.165"></a>
-<span class="sourceLineNo">166</span>   */<a name="line.166"></a>
-<span class="sourceLineNo">167</span>  public void setMinMaxModifiedProcIds(long min, long max) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    this.minModifiedProcId = min;<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    this.maxModifiedProcId = max;<a name="line.169"></a>
-<span class="sourceLineNo">170</span>  }<a name="line.170"></a>
-<span class="sourceLineNo">171</span>  /**<a name="line.171"></a>
-<span class="sourceLineNo">172</span>   * This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The<a name="line.172"></a>
-<span class="sourceLineNo">173</span>   * {@link #delete(long)} method above assume that the {@link BitSetNode} exists, but when restart<a name="line.173"></a>
-<span class="sourceLineNo">174</span>   * this is not true, as we will read the wal files in reverse order so a delete may come first.<a name="line.174"></a>
-<span class="sourceLineNo">175</span>   */<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  public void setDeleted(long procId, boolean isDeleted) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    BitSetNode node = getOrCreateNode(procId);<a name="line.177"></a>
-<span class="sourceLineNo">178</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node=" + node;<a name="line.178"></a>
-<span class="sourceLineNo">179</span>    node.updateState(procId, isDeleted);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    trackProcIds(procId);<a name="line.180"></a>
-<span class="sourceLineNo">181</span>  }<a name="line.181"></a>
-<span class="sourceLineNo">182</span><a name="line.182"></a>
-<span class="sourceLineNo">183</span>  /**<a name="line.183"></a>
-<span class="sourceLineNo">184</span>   * Set the given bit for the procId to delete if it was modified before.<a name="line.184"></a>
-<span class="sourceLineNo">185</span>   * &lt;p/&gt;<a name="line.185"></a>
-<span class="sourceLineNo">186</span>   * This method is used to test whether a procedure wal file can be safely deleted, as if all the<a name="line.186"></a>
-<span class="sourceLineNo">187</span>   * procedures in the given procedure wal file has been modified in the new procedure wal files,<a name="line.187"></a>
-<span class="sourceLineNo">188</span>   * then we can delete it.<a name="line.188"></a>
-<span class="sourceLineNo">189</span>   */<a name="line.189"></a>
-<span class="sourceLineNo">190</span>  public void setDeletedIfModified(long... procId) {<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    BitSetNode node = null;<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    for (int i = 0; i &lt; procId.length; ++i) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      node = lookupClosestNode(node, procId[i]);<a name="line.193"></a>
-<span class="sourceLineNo">194</span>      if (node != null &amp;&amp; node.isModified(procId[i])) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span>        node.delete(procId[i]);<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      }<a name="line.196"></a>
-<span class="sourceLineNo">197</span>    }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>  }<a name="line.198"></a>
-<span class="sourceLineNo">199</span><a name="line.199"></a>
-<span class="sourceLineNo">200</span>  /**<a name="line.200"></a>
-<span class="sourceLineNo">201</span>   * Similar with {@link #setDeletedIfModified(long...)}, but here the {@code procId} are given by<a name="line.201"></a>
-<span class="sourceLineNo">202</span>   * the {@code tracker}. If a procedure is modified by us, and also by the given {@code tracker},<a name="line.202"></a>
-<span class="sourceLineNo">203</span>   * then we mark it as deleted.<a name="line.203"></a>
-<span class="sourceLineNo">204</span>   * @see #setDeletedIfModified(long...)<a name="line.204"></a>
-<span class="sourceLineNo">205</span>   */<a name="line.205"></a>
-<span class="sourceLineNo">206</span>  public void setDeletedIfModifiedInBoth(ProcedureStoreTracker tracker, boolean globalTracker) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    BitSetNode trackerNode = null;<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    for (BitSetNode node : map.values()) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>      final long minProcId = node.getStart();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      final long maxProcId = node.getEnd();<a name="line.210"></a>
-<span class="sourceLineNo">211</span>      for (long procId = minProcId; procId &lt;= maxProcId; ++procId) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span>        if (!node.isModified(procId)) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>          continue;<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        }<a name="line.214"></a>
-<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span>        trackerNode = tracker.lookupClosestNode(trackerNode, procId);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        if (trackerNode == null || !trackerNode.contains(procId)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span>          // the procId is not exist in the track, we can only delete the proc<a name="line.218"></a>
-<span class="sourceLineNo">219</span>          // if globalTracker set to true.<a name="line.219"></a>
-<span class="sourceLineNo">220</span>          // Only if the procedure is not in the global tracker we can delete the<a name="line.220"></a>
-<span class="sourceLineNo">221</span>          // the procedure. In other cases, the procedure may not update in a single<a name="line.221"></a>
-<span class="sourceLineNo">222</span>          // log, we cannot delete it just because the log's track doesn't have<a name="line.222"></a>
-<span class="sourceLineNo">223</span>          // any info for the procedure.<a name="line.223"></a>
-<span class="sourceLineNo">224</span>          if (globalTracker) {<a name="line.224"></a>
-<span class="sourceLineNo">225</span>            node.delete(procId);<a name="line.225"></a>
-<span class="sourceLineNo">226</span>          }<a name="line.226"></a>
-<span class="sourceLineNo">227</span>          continue;<a name="line.227"></a>
-<span class="sourceLineNo">228</span>        }<a name="line.228"></a>
-<span class="sourceLineNo">229</span>        // Only check delete in the global tracker, only global tracker has the<a name="line.229"></a>
-<span class="sourceLineNo">230</span>        // whole picture<a name="line.230"></a>
-<span class="sourceLineNo">231</span>        if (globalTracker &amp;&amp; trackerNode.isDeleted(procId) == DeleteState.YES) {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>          node.delete(procId);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>          continue;<a name="line.233"></a>
-<span class="sourceLineNo">234</span>        }<a name="line.234"></a>
-<span class="sourceLineNo">235</span>        if (trackerNode.isModified(procId)) {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>          // the procedure was modified<a name="line.236"></a>
-<span class="sourceLineNo">237</span>          node.delete(procId);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        }<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      }<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    }<a name="line.240"></a>
+<span class="sourceLineNo">164</span>    trackProcIds(procId);<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    return node;<a name="line.165"></a>
+<span class="sourceLineNo">166</span>  }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span>  /**<a name="line.168"></a>
+<span class="sourceLineNo">169</span>   * Will be called when restarting where we need to rebuild the ProcedureStoreTracker.<a name="line.169"></a>
+<span class="sourceLineNo">170</span>   */<a name="line.170"></a>
+<span class="sourceLineNo">171</span>  public void setMinMaxModifiedProcIds(long min, long max) {<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    this.minModifiedProcId = min;<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    this.maxModifiedProcId = max;<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  }<a name="line.174"></a>
+<span class="sourceLineNo">175</span>  /**<a name="line.175"></a>
+<span class="sourceLineNo">176</span>   * This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The<a name="line.176"></a>
+<span class="sourceLineNo">177</span>   * {@link #delete(long)} method above assume that the {@link BitSetNode} exists, but when restart<a name="line.177"></a>
+<span class="sourceLineNo">178</span>   * this is not true, as we will read the wal files in reverse order so a delete may come first.<a name="line.178"></a>
+<span class="sourceLineNo">179</span>   */<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  public void setDeleted(long procId, boolean isDeleted) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    BitSetNode node = getOrCreateNode(procId);<a name="line.181"></a>
+<span class="sourceLineNo">182</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node=" + node;<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    node.updateState(procId, isDeleted);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    trackProcIds(procId);<a name="line.184"></a>
+<span class="sourceLineNo">185</span>  }<a name="line.185"></a>
+<span class="sourceLineNo">186</span><a name="line.186"></a>
+<span class="sourceLineNo">187</span>  /**<a name="line.187"></a>
+<span class="sourceLineNo">188</span>   * Set the given bit for the procId to delete if it was modified before.<a name="line.188"></a>
+<span class="sourceLineNo">189</span>   * &lt;p/&gt;<a name="line.189"></a>
+<span class="sourceLineNo">190</span>   * This method is used to test whether a procedure wal file can be safely deleted, as if all the<a name="line.190"></a>
+<span class="sourceLineNo">191</span>   * procedures in the given procedure wal file has been modified in the new procedure wal files,<a name="line.191"></a>
+<span class="sourceLineNo">192</span>   * then we can delete it.<a name="line.192"></a>
+<span class="sourceLineNo">193</span>   */<a name="line.193"></a>
+<span class="sourceLineNo">194</span>  public void setDeletedIfModified(long... procId) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    BitSetNode node = null;<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    for (int i = 0; i &lt; procId.length; ++i) {<a name="line.196"></a>
+<span class="sourceLineNo">197</span>      node = lookupClosestNode(node, procId[i]);<a name="line.197"></a>
+<span class="sourceLineNo">198</span>      if (node != null &amp;&amp; node.isModified(procId[i])) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>        node.delete(procId[i]);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      }<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    }<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  private void setDeleteIf(ProcedureStoreTracker tracker,<a name="line.204"></a>
+<span class="sourceLineNo">205</span>      BiFunction&lt;BitSetNode, Long, Boolean&gt; func) {<a name="line.205"></a>
+<span class="sourceLineNo">206</span>    BitSetNode trackerNode = null;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    for (BitSetNode node : map.values()) {<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      long minProcId = node.getStart();<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      long maxProcId = node.getEnd();<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      for (long procId = minProcId; procId &lt;= maxProcId; ++procId) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>        if (!node.isModified(procId)) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>          continue;<a name="line.212"></a>
+<span class="sourceLineNo">213</span>        }<a name="line.213"></a>
+<span class="sourceLineNo">214</span><a name="line.214"></a>
+<span class="sourceLineNo">215</span>        trackerNode = tracker.lookupClosestNode(trackerNode, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>        if (func.apply(trackerNode, procId)) {<a name="line.216"></a>
+<span class="sourceLineNo">217</span>          node.delete(procId);<a name="line.217"></a>
+<span class="sourceLineNo">218</span>        }<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      }<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>  }<a name="line.221"></a>
+<span class="sourceLineNo">222</span><a name="line.222"></a>
+<span class="sourceLineNo">223</span>  /**<a name="line.223"></a>
+<span class="sourceLineNo">224</span>   * For the global tracker, we will use this method to build the holdingCleanupTracker, as the<a name="line.224"></a>
+<span class="sourceLineNo">225</span>   * modified flags will be cleared after rolling so we only need to test the deleted flags.<a name="line.225"></a>
+<span class="sourceLineNo">226</span>   * @see #setDeletedIfModifiedInBoth(ProcedureStoreTracker)<a name="line.226"></a>
+<span class="sourceLineNo">227</span>   */<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  public void setDeletedIfDeletedByThem(ProcedureStoreTracker tracker) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span>    setDeleteIf(tracker, (node, procId) -&gt; node == null || !node.contains(procId) ||<a name="line.229"></a>
+<span class="sourceLineNo">230</span>      node.isDeleted(procId) == DeleteState.YES);<a name="line.230"></a>
+<span class="sourceLineNo">231</span>  }<a name="line.231"></a>
+<span class="sourceLineNo">232</span><a name="line.232"></a>
+<span class="sourceLineNo">233</span>  /**<a name="line.233"></a>
+<span class="sourceLineNo">234</span>   * Similar with {@link #setDeletedIfModified(long...)}, but here the {@code procId} are given by<a name="line.234"></a>
+<span class="sourceLineNo">235</span>   * the {@code tracker}. If a procedure is modified by us, and also by the given {@code tracker},<a name="line.235"></a>
+<span class="sourceLineNo">236</span>   * then we mark it as deleted.<a name="line.236"></a>
+<span class="sourceLineNo">237</span>   * @see #setDeletedIfModified(long...)<a name="line.237"></a>
+<span class="sourceLineNo">238</span>   */<a name="line.238"></a>
+<span class="sourceLineNo">239</span>  public void setDeletedIfModifiedInBoth(ProcedureStoreTracker tracker) {<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    setDeleteIf(tracker, (node, procId) -&gt; node != null &amp;&amp; node.isModified(procId));<a name="line.240"></a>
 <span class="sourceLineNo">241</span>  }<a name="line.241"></a>
 <span class="sourceLineNo">242</span><a name="line.242"></a>
 <span class="sourceLineNo">243</span>  /**<a name="line.243"></a>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
index 1579eab..dc5908e 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html
@@ -31,221 +31,221 @@
 <span class="sourceLineNo">023</span>import java.util.Iterator;<a name="line.23"></a>
 <span class="sourceLineNo">024</span>import java.util.Map;<a name="line.24"></a>
 <span class="sourceLineNo">025</span>import java.util.TreeMap;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import java.util.stream.LongStream;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import org.apache.hadoop.hbase.procedure2.Procedure;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.29"></a>
-<span class="sourceLineNo">030</span><a name="line.30"></a>
-<span class="sourceLineNo">031</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;<a name="line.31"></a>
-<span class="sourceLineNo">032</span><a name="line.32"></a>
-<span class="sourceLineNo">033</span>/**<a name="line.33"></a>
-<span class="sourceLineNo">034</span> * Keeps track of live procedures.<a name="line.34"></a>
-<span class="sourceLineNo">035</span> *<a name="line.35"></a>
-<span class="sourceLineNo">036</span> * It can be used by the ProcedureStore to identify which procedures are already<a name="line.36"></a>
-<span class="sourceLineNo">037</span> * deleted/completed to avoid the deserialization step on restart<a name="line.37"></a>
-<span class="sourceLineNo">038</span> */<a name="line.38"></a>
-<span class="sourceLineNo">039</span>@InterfaceAudience.Private<a name="line.39"></a>
-<span class="sourceLineNo">040</span>@InterfaceStability.Evolving<a name="line.40"></a>
-<span class="sourceLineNo">041</span>public class ProcedureStoreTracker {<a name="line.41"></a>
-<span class="sourceLineNo">042</span>  // Key is procedure id corresponding to first bit of the bitmap.<a name="line.42"></a>
-<span class="sourceLineNo">043</span>  private final TreeMap&lt;Long, BitSetNode&gt; map = new TreeMap&lt;&gt;();<a name="line.43"></a>
-<span class="sourceLineNo">044</span><a name="line.44"></a>
-<span class="sourceLineNo">045</span>  /**<a name="line.45"></a>
-<span class="sourceLineNo">046</span>   * If true, do not remove bits corresponding to deleted procedures. Note that this can result<a name="line.46"></a>
-<span class="sourceLineNo">047</span>   * in huge bitmaps overtime.<a name="line.47"></a>
-<span class="sourceLineNo">048</span>   * Currently, it's set to true only when building tracker state from logs during recovery. During<a name="line.48"></a>
-<span class="sourceLineNo">049</span>   * recovery, if we are sure that a procedure has been deleted, reading its old update entries<a name="line.49"></a>
-<span class="sourceLineNo">050</span>   * can be skipped.<a name="line.50"></a>
-<span class="sourceLineNo">051</span>   */<a name="line.51"></a>
-<span class="sourceLineNo">052</span>  private boolean keepDeletes = false;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>  /**<a name="line.53"></a>
-<span class="sourceLineNo">054</span>   * If true, it means tracker has incomplete information about the active/deleted procedures.<a name="line.54"></a>
-<span class="sourceLineNo">055</span>   * It's set to true only when recovering from old logs. See {@link #isDeleted(long)} docs to<a name="line.55"></a>
-<span class="sourceLineNo">056</span>   * understand it's real use.<a name="line.56"></a>
-<span class="sourceLineNo">057</span>   */<a name="line.57"></a>
-<span class="sourceLineNo">058</span>  boolean partial = false;<a name="line.58"></a>
-<span class="sourceLineNo">059</span><a name="line.59"></a>
-<span class="sourceLineNo">060</span>  private long minModifiedProcId = Long.MAX_VALUE;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>  private long maxModifiedProcId = Long.MIN_VALUE;<a name="line.61"></a>
-<span class="sourceLineNo">062</span><a name="line.62"></a>
-<span class="sourceLineNo">063</span>  public enum DeleteState { YES, NO, MAYBE }<a name="line.63"></a>
-<span class="sourceLineNo">064</span><a name="line.64"></a>
-<span class="sourceLineNo">065</span>  public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) {<a name="line.65"></a>
-<span class="sourceLineNo">066</span>    reset();<a name="line.66"></a>
-<span class="sourceLineNo">067</span>    for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) {<a name="line.67"></a>
-<span class="sourceLineNo">068</span>      final BitSetNode node = new BitSetNode(protoNode);<a name="line.68"></a>
-<span class="sourceLineNo">069</span>      map.put(node.getStart(), node);<a name="line.69"></a>
-<span class="sourceLineNo">070</span>    }<a name="line.70"></a>
-<span class="sourceLineNo">071</span>  }<a name="line.71"></a>
-<span class="sourceLineNo">072</span><a name="line.72"></a>
-<span class="sourceLineNo">073</span>  /**<a name="line.73"></a>
-<span class="sourceLineNo">074</span>   * Resets internal state to same as given {@code tracker}. Does deep copy of the bitmap.<a name="line.74"></a>
-<span class="sourceLineNo">075</span>   */<a name="line.75"></a>
-<span class="sourceLineNo">076</span>  public void resetTo(ProcedureStoreTracker tracker) {<a name="line.76"></a>
-<span class="sourceLineNo">077</span>    resetTo(tracker, false);<a name="line.77"></a>
-<span class="sourceLineNo">078</span>  }<a name="line.78"></a>
-<span class="sourceLineNo">079</span><a name="line.79"></a>
-<span class="sourceLineNo">080</span>  /**<a name="line.80"></a>
-<span class="sourceLineNo">081</span>   * Resets internal state to same as given {@code tracker}, and change the deleted flag according<a name="line.81"></a>
-<span class="sourceLineNo">082</span>   * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap.<a name="line.82"></a>
-<span class="sourceLineNo">083</span>   * &lt;p/&gt;<a name="line.83"></a>
-<span class="sourceLineNo">084</span>   * The {@code resetDelete} will be set to true when building cleanup tracker, please see the<a name="line.84"></a>
-<span class="sourceLineNo">085</span>   * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the<a name="line.85"></a>
-<span class="sourceLineNo">086</span>   * deleted flag if {@code resetDelete} is true.<a name="line.86"></a>
-<span class="sourceLineNo">087</span>   */<a name="line.87"></a>
-<span class="sourceLineNo">088</span>  public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) {<a name="line.88"></a>
-<span class="sourceLineNo">089</span>    reset();<a name="line.89"></a>
-<span class="sourceLineNo">090</span>    this.partial = tracker.partial;<a name="line.90"></a>
-<span class="sourceLineNo">091</span>    this.minModifiedProcId = tracker.minModifiedProcId;<a name="line.91"></a>
-<span class="sourceLineNo">092</span>    this.maxModifiedProcId = tracker.maxModifiedProcId;<a name="line.92"></a>
-<span class="sourceLineNo">093</span>    this.keepDeletes = tracker.keepDeletes;<a name="line.93"></a>
-<span class="sourceLineNo">094</span>    for (Map.Entry&lt;Long, BitSetNode&gt; entry : tracker.map.entrySet()) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span>      map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete));<a name="line.95"></a>
-<span class="sourceLineNo">096</span>    }<a name="line.96"></a>
-<span class="sourceLineNo">097</span>  }<a name="line.97"></a>
-<span class="sourceLineNo">098</span><a name="line.98"></a>
-<span class="sourceLineNo">099</span>  public void insert(long procId) {<a name="line.99"></a>
-<span class="sourceLineNo">100</span>    insert(null, procId);<a name="line.100"></a>
+<span class="sourceLineNo">026</span>import java.util.function.BiFunction;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import java.util.stream.LongStream;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.procedure2.Procedure;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;<a name="line.32"></a>
+<span class="sourceLineNo">033</span><a name="line.33"></a>
+<span class="sourceLineNo">034</span>/**<a name="line.34"></a>
+<span class="sourceLineNo">035</span> * Keeps track of live procedures.<a name="line.35"></a>
+<span class="sourceLineNo">036</span> *<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * It can be used by the ProcedureStore to identify which procedures are already<a name="line.37"></a>
+<span class="sourceLineNo">038</span> * deleted/completed to avoid the deserialization step on restart<a name="line.38"></a>
+<span class="sourceLineNo">039</span> */<a name="line.39"></a>
+<span class="sourceLineNo">040</span>@InterfaceAudience.Private<a name="line.40"></a>
+<span class="sourceLineNo">041</span>@InterfaceStability.Evolving<a name="line.41"></a>
+<span class="sourceLineNo">042</span>public class ProcedureStoreTracker {<a name="line.42"></a>
+<span class="sourceLineNo">043</span>  // Key is procedure id corresponding to first bit of the bitmap.<a name="line.43"></a>
+<span class="sourceLineNo">044</span>  private final TreeMap&lt;Long, BitSetNode&gt; map = new TreeMap&lt;&gt;();<a name="line.44"></a>
+<span class="sourceLineNo">045</span><a name="line.45"></a>
+<span class="sourceLineNo">046</span>  /**<a name="line.46"></a>
+<span class="sourceLineNo">047</span>   * If true, do not remove bits corresponding to deleted procedures. Note that this can result<a name="line.47"></a>
+<span class="sourceLineNo">048</span>   * in huge bitmaps overtime.<a name="line.48"></a>
+<span class="sourceLineNo">049</span>   * Currently, it's set to true only when building tracker state from logs during recovery. During<a name="line.49"></a>
+<span class="sourceLineNo">050</span>   * recovery, if we are sure that a procedure has been deleted, reading its old update entries<a name="line.50"></a>
+<span class="sourceLineNo">051</span>   * can be skipped.<a name="line.51"></a>
+<span class="sourceLineNo">052</span>   */<a name="line.52"></a>
+<span class="sourceLineNo">053</span>  private boolean keepDeletes = false;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>  /**<a name="line.54"></a>
+<span class="sourceLineNo">055</span>   * If true, it means tracker has incomplete information about the active/deleted procedures.<a name="line.55"></a>
+<span class="sourceLineNo">056</span>   * It's set to true only when recovering from old logs. See {@link #isDeleted(long)} docs to<a name="line.56"></a>
+<span class="sourceLineNo">057</span>   * understand it's real use.<a name="line.57"></a>
+<span class="sourceLineNo">058</span>   */<a name="line.58"></a>
+<span class="sourceLineNo">059</span>  boolean partial = false;<a name="line.59"></a>
+<span class="sourceLineNo">060</span><a name="line.60"></a>
+<span class="sourceLineNo">061</span>  private long minModifiedProcId = Long.MAX_VALUE;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>  private long maxModifiedProcId = Long.MIN_VALUE;<a name="line.62"></a>
+<span class="sourceLineNo">063</span><a name="line.63"></a>
+<span class="sourceLineNo">064</span>  public enum DeleteState { YES, NO, MAYBE }<a name="line.64"></a>
+<span class="sourceLineNo">065</span><a name="line.65"></a>
+<span class="sourceLineNo">066</span>  public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) {<a name="line.66"></a>
+<span class="sourceLineNo">067</span>    reset();<a name="line.67"></a>
+<span class="sourceLineNo">068</span>    for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) {<a name="line.68"></a>
+<span class="sourceLineNo">069</span>      final BitSetNode node = new BitSetNode(protoNode);<a name="line.69"></a>
+<span class="sourceLineNo">070</span>      map.put(node.getStart(), node);<a name="line.70"></a>
+<span class="sourceLineNo">071</span>    }<a name="line.71"></a>
+<span class="sourceLineNo">072</span>  }<a name="line.72"></a>
+<span class="sourceLineNo">073</span><a name="line.73"></a>
+<span class="sourceLineNo">074</span>  /**<a name="line.74"></a>
+<span class="sourceLineNo">075</span>   * Resets internal state to same as given {@code tracker}. Does deep copy of the bitmap.<a name="line.75"></a>
+<span class="sourceLineNo">076</span>   */<a name="line.76"></a>
+<span class="sourceLineNo">077</span>  public void resetTo(ProcedureStoreTracker tracker) {<a name="line.77"></a>
+<span class="sourceLineNo">078</span>    resetTo(tracker, false);<a name="line.78"></a>
+<span class="sourceLineNo">079</span>  }<a name="line.79"></a>
+<span class="sourceLineNo">080</span><a name="line.80"></a>
+<span class="sourceLineNo">081</span>  /**<a name="line.81"></a>
+<span class="sourceLineNo">082</span>   * Resets internal state to same as given {@code tracker}, and change the deleted flag according<a name="line.82"></a>
+<span class="sourceLineNo">083</span>   * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap.<a name="line.83"></a>
+<span class="sourceLineNo">084</span>   * &lt;p/&gt;<a name="line.84"></a>
+<span class="sourceLineNo">085</span>   * The {@code resetDelete} will be set to true when building cleanup tracker, please see the<a name="line.85"></a>
+<span class="sourceLineNo">086</span>   * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the<a name="line.86"></a>
+<span class="sourceLineNo">087</span>   * deleted flag if {@code resetDelete} is true.<a name="line.87"></a>
+<span class="sourceLineNo">088</span>   */<a name="line.88"></a>
+<span class="sourceLineNo">089</span>  public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) {<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    reset();<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    // resetDelete will true if we are building the cleanup tracker, as we will reset deleted flags<a name="line.91"></a>
+<span class="sourceLineNo">092</span>    // for all the unmodified bits to 1, the partial flag is useless so set it to false for not<a name="line.92"></a>
+<span class="sourceLineNo">093</span>    // confusing the developers when debugging.<a name="line.93"></a>
+<span class="sourceLineNo">094</span>    this.partial = resetDelete ? false : tracker.partial;<a name="line.94"></a>
+<span class="sourceLineNo">095</span>    this.minModifiedProcId = tracker.minModifiedProcId;<a name="line.95"></a>
+<span class="sourceLineNo">096</span>    this.maxModifiedProcId = tracker.maxModifiedProcId;<a name="line.96"></a>
+<span class="sourceLineNo">097</span>    this.keepDeletes = tracker.keepDeletes;<a name="line.97"></a>
+<span class="sourceLineNo">098</span>    for (Map.Entry&lt;Long, BitSetNode&gt; entry : tracker.map.entrySet()) {<a name="line.98"></a>
+<span class="sourceLineNo">099</span>      map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete));<a name="line.99"></a>
+<span class="sourceLineNo">100</span>    }<a name="line.100"></a>
 <span class="sourceLineNo">101</span>  }<a name="line.101"></a>
 <span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span>  public void insert(long[] procIds) {<a name="line.103"></a>
-<span class="sourceLineNo">104</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>      insert(procIds[i]);<a name="line.105"></a>
-<span class="sourceLineNo">106</span>    }<a name="line.106"></a>
-<span class="sourceLineNo">107</span>  }<a name="line.107"></a>
-<span class="sourceLineNo">108</span><a name="line.108"></a>
-<span class="sourceLineNo">109</span>  public void insert(long procId, long[] subProcIds) {<a name="line.109"></a>
-<span class="sourceLineNo">110</span>    BitSetNode node = update(null, procId);<a name="line.110"></a>
-<span class="sourceLineNo">111</span>    for (int i = 0; i &lt; subProcIds.length; ++i) {<a name="line.111"></a>
-<span class="sourceLineNo">112</span>      node = insert(node, subProcIds[i]);<a name="line.112"></a>
-<span class="sourceLineNo">113</span>    }<a name="line.113"></a>
-<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span>  private BitSetNode insert(BitSetNode node, long procId) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>    if (node == null || !node.contains(procId)) {<a name="line.117"></a>
-<span class="sourceLineNo">118</span>      node = getOrCreateNode(procId);<a name="line.118"></a>
-<span class="sourceLineNo">119</span>    }<a name="line.119"></a>
-<span class="sourceLineNo">120</span>    node.insertOrUpdate(procId);<a name="line.120"></a>
-<span class="sourceLineNo">121</span>    trackProcIds(procId);<a name="line.121"></a>
-<span class="sourceLineNo">122</span>    return node;<a name="line.122"></a>
-<span class="sourceLineNo">123</span>  }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span>  public void update(long procId) {<a name="line.125"></a>
-<span class="sourceLineNo">126</span>    update(null, procId);<a name="line.126"></a>
+<span class="sourceLineNo">103</span>  public void insert(long procId) {<a name="line.103"></a>
+<span class="sourceLineNo">104</span>    insert(null, procId);<a name="line.104"></a>
+<span class="sourceLineNo">105</span>  }<a name="line.105"></a>
+<span class="sourceLineNo">106</span><a name="line.106"></a>
+<span class="sourceLineNo">107</span>  public void insert(long[] procIds) {<a name="line.107"></a>
+<span class="sourceLineNo">108</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span>      insert(procIds[i]);<a name="line.109"></a>
+<span class="sourceLineNo">110</span>    }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>  }<a name="line.111"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span>  public void insert(long procId, long[] subProcIds) {<a name="line.113"></a>
+<span class="sourceLineNo">114</span>    BitSetNode node = update(null, procId);<a name="line.114"></a>
+<span class="sourceLineNo">115</span>    for (int i = 0; i &lt; subProcIds.length; ++i) {<a name="line.115"></a>
+<span class="sourceLineNo">116</span>      node = insert(node, subProcIds[i]);<a name="line.116"></a>
+<span class="sourceLineNo">117</span>    }<a name="line.117"></a>
+<span class="sourceLineNo">118</span>  }<a name="line.118"></a>
+<span class="sourceLineNo">119</span><a name="line.119"></a>
+<span class="sourceLineNo">120</span>  private BitSetNode insert(BitSetNode node, long procId) {<a name="line.120"></a>
+<span class="sourceLineNo">121</span>    if (node == null || !node.contains(procId)) {<a name="line.121"></a>
+<span class="sourceLineNo">122</span>      node = getOrCreateNode(procId);<a name="line.122"></a>
+<span class="sourceLineNo">123</span>    }<a name="line.123"></a>
+<span class="sourceLineNo">124</span>    node.insertOrUpdate(procId);<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    trackProcIds(procId);<a name="line.125"></a>
+<span class="sourceLineNo">126</span>    return node;<a name="line.126"></a>
 <span class="sourceLineNo">127</span>  }<a name="line.127"></a>
 <span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span>  private BitSetNode update(BitSetNode node, long procId) {<a name="line.129"></a>
-<span class="sourceLineNo">130</span>    node = lookupClosestNode(node, procId);<a name="line.130"></a>
-<span class="sourceLineNo">131</span>    assert node != null : "expected node to update procId=" + procId;<a name="line.131"></a>
-<span class="sourceLineNo">132</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.132"></a>
-<span class="sourceLineNo">133</span>    node.insertOrUpdate(procId);<a name="line.133"></a>
-<span class="sourceLineNo">134</span>    trackProcIds(procId);<a name="line.134"></a>
-<span class="sourceLineNo">135</span>    return node;<a name="line.135"></a>
-<span class="sourceLineNo">136</span>  }<a name="line.136"></a>
-<span class="sourceLineNo">137</span><a name="line.137"></a>
-<span class="sourceLineNo">138</span>  public void delete(long procId) {<a name="line.138"></a>
-<span class="sourceLineNo">139</span>    delete(null, procId);<a name="line.139"></a>
+<span class="sourceLineNo">129</span>  public void update(long procId) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span>    update(null, procId);<a name="line.130"></a>
+<span class="sourceLineNo">131</span>  }<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span>  private BitSetNode update(BitSetNode node, long procId) {<a name="line.133"></a>
+<span class="sourceLineNo">134</span>    node = lookupClosestNode(node, procId);<a name="line.134"></a>
+<span class="sourceLineNo">135</span>    assert node != null : "expected node to update procId=" + procId;<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.136"></a>
+<span class="sourceLineNo">137</span>    node.insertOrUpdate(procId);<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    trackProcIds(procId);<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return node;<a name="line.139"></a>
 <span class="sourceLineNo">140</span>  }<a name="line.140"></a>
 <span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span>  public void delete(final long[] procIds) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>    Arrays.sort(procIds);<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    BitSetNode node = null;<a name="line.144"></a>
-<span class="sourceLineNo">145</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.145"></a>
-<span class="sourceLineNo">146</span>      node = delete(node, procIds[i]);<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    }<a name="line.147"></a>
-<span class="sourceLineNo">148</span>  }<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span>  private BitSetNode delete(BitSetNode node, long procId) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>    node = lookupClosestNode(node, procId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    assert node != null : "expected node to delete procId=" + procId;<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    node.delete(procId);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    if (!keepDeletes &amp;&amp; node.isEmpty()) {<a name="line.155"></a>
-<span class="sourceLineNo">156</span>      // TODO: RESET if (map.size() == 1)<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      map.remove(node.getStart());<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span>    trackProcIds(procId);<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    return node;<a name="line.161"></a>
-<span class="sourceLineNo">162</span>  }<a name="line.162"></a>
+<span class="sourceLineNo">142</span>  public void delete(long procId) {<a name="line.142"></a>
+<span class="sourceLineNo">143</span>    delete(null, procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span>  }<a name="line.144"></a>
+<span class="sourceLineNo">145</span><a name="line.145"></a>
+<span class="sourceLineNo">146</span>  public void delete(final long[] procIds) {<a name="line.146"></a>
+<span class="sourceLineNo">147</span>    Arrays.sort(procIds);<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    BitSetNode node = null;<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    for (int i = 0; i &lt; procIds.length; ++i) {<a name="line.149"></a>
+<span class="sourceLineNo">150</span>      node = delete(node, procIds[i]);<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    }<a name="line.151"></a>
+<span class="sourceLineNo">152</span>  }<a name="line.152"></a>
+<span class="sourceLineNo">153</span><a name="line.153"></a>
+<span class="sourceLineNo">154</span>  private BitSetNode delete(BitSetNode node, long procId) {<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    node = lookupClosestNode(node, procId);<a name="line.155"></a>
+<span class="sourceLineNo">156</span>    assert node != null : "expected node to delete procId=" + procId;<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node";<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    node.delete(procId);<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    if (!keepDeletes &amp;&amp; node.isEmpty()) {<a name="line.159"></a>
+<span class="sourceLineNo">160</span>      // TODO: RESET if (map.size() == 1)<a name="line.160"></a>
+<span class="sourceLineNo">161</span>      map.remove(node.getStart());<a name="line.161"></a>
+<span class="sourceLineNo">162</span>    }<a name="line.162"></a>
 <span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span>  /**<a name="line.164"></a>
-<span class="sourceLineNo">165</span>   * Will be called when restarting where we need to rebuild the ProcedureStoreTracker.<a name="line.165"></a>
-<span class="sourceLineNo">166</span>   */<a name="line.166"></a>
-<span class="sourceLineNo">167</span>  public void setMinMaxModifiedProcIds(long min, long max) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    this.minModifiedProcId = min;<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    this.maxModifiedProcId = max;<a name="line.169"></a>
-<span class="sourceLineNo">170</span>  }<a name="line.170"></a>
-<span class="sourceLineNo">171</span>  /**<a name="line.171"></a>
-<span class="sourceLineNo">172</span>   * This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The<a name="line.172"></a>
-<span class="sourceLineNo">173</span>   * {@link #delete(long)} method above assume that the {@link BitSetNode} exists, but when restart<a name="line.173"></a>
-<span class="sourceLineNo">174</span>   * this is not true, as we will read the wal files in reverse order so a delete may come first.<a name="line.174"></a>
-<span class="sourceLineNo">175</span>   */<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  public void setDeleted(long procId, boolean isDeleted) {<a name="line.176"></a>
-<span class="sourceLineNo">177</span>    BitSetNode node = getOrCreateNode(procId);<a name="line.177"></a>
-<span class="sourceLineNo">178</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node=" + node;<a name="line.178"></a>
-<span class="sourceLineNo">179</span>    node.updateState(procId, isDeleted);<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    trackProcIds(procId);<a name="line.180"></a>
-<span class="sourceLineNo">181</span>  }<a name="line.181"></a>
-<span class="sourceLineNo">182</span><a name="line.182"></a>
-<span class="sourceLineNo">183</span>  /**<a name="line.183"></a>
-<span class="sourceLineNo">184</span>   * Set the given bit for the procId to delete if it was modified before.<a name="line.184"></a>
-<span class="sourceLineNo">185</span>   * &lt;p/&gt;<a name="line.185"></a>
-<span class="sourceLineNo">186</span>   * This method is used to test whether a procedure wal file can be safely deleted, as if all the<a name="line.186"></a>
-<span class="sourceLineNo">187</span>   * procedures in the given procedure wal file has been modified in the new procedure wal files,<a name="line.187"></a>
-<span class="sourceLineNo">188</span>   * then we can delete it.<a name="line.188"></a>
-<span class="sourceLineNo">189</span>   */<a name="line.189"></a>
-<span class="sourceLineNo">190</span>  public void setDeletedIfModified(long... procId) {<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    BitSetNode node = null;<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    for (int i = 0; i &lt; procId.length; ++i) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span>      node = lookupClosestNode(node, procId[i]);<a name="line.193"></a>
-<span class="sourceLineNo">194</span>      if (node != null &amp;&amp; node.isModified(procId[i])) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span>        node.delete(procId[i]);<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      }<a name="line.196"></a>
-<span class="sourceLineNo">197</span>    }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>  }<a name="line.198"></a>
-<span class="sourceLineNo">199</span><a name="line.199"></a>
-<span class="sourceLineNo">200</span>  /**<a name="line.200"></a>
-<span class="sourceLineNo">201</span>   * Similar with {@link #setDeletedIfModified(long...)}, but here the {@code procId} are given by<a name="line.201"></a>
-<span class="sourceLineNo">202</span>   * the {@code tracker}. If a procedure is modified by us, and also by the given {@code tracker},<a name="line.202"></a>
-<span class="sourceLineNo">203</span>   * then we mark it as deleted.<a name="line.203"></a>
-<span class="sourceLineNo">204</span>   * @see #setDeletedIfModified(long...)<a name="line.204"></a>
-<span class="sourceLineNo">205</span>   */<a name="line.205"></a>
-<span class="sourceLineNo">206</span>  public void setDeletedIfModifiedInBoth(ProcedureStoreTracker tracker, boolean globalTracker) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    BitSetNode trackerNode = null;<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    for (BitSetNode node : map.values()) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>      final long minProcId = node.getStart();<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      final long maxProcId = node.getEnd();<a name="line.210"></a>
-<span class="sourceLineNo">211</span>      for (long procId = minProcId; procId &lt;= maxProcId; ++procId) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span>        if (!node.isModified(procId)) {<a name="line.212"></a>
-<span class="sourceLineNo">213</span>          continue;<a name="line.213"></a>
-<span class="sourceLineNo">214</span>        }<a name="line.214"></a>
-<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span>        trackerNode = tracker.lookupClosestNode(trackerNode, procId);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>        if (trackerNode == null || !trackerNode.contains(procId)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span>          // the procId is not exist in the track, we can only delete the proc<a name="line.218"></a>
-<span class="sourceLineNo">219</span>          // if globalTracker set to true.<a name="line.219"></a>
-<span class="sourceLineNo">220</span>          // Only if the procedure is not in the global tracker we can delete the<a name="line.220"></a>
-<span class="sourceLineNo">221</span>          // the procedure. In other cases, the procedure may not update in a single<a name="line.221"></a>
-<span class="sourceLineNo">222</span>          // log, we cannot delete it just because the log's track doesn't have<a name="line.222"></a>
-<span class="sourceLineNo">223</span>          // any info for the procedure.<a name="line.223"></a>
-<span class="sourceLineNo">224</span>          if (globalTracker) {<a name="line.224"></a>
-<span class="sourceLineNo">225</span>            node.delete(procId);<a name="line.225"></a>
-<span class="sourceLineNo">226</span>          }<a name="line.226"></a>
-<span class="sourceLineNo">227</span>          continue;<a name="line.227"></a>
-<span class="sourceLineNo">228</span>        }<a name="line.228"></a>
-<span class="sourceLineNo">229</span>        // Only check delete in the global tracker, only global tracker has the<a name="line.229"></a>
-<span class="sourceLineNo">230</span>        // whole picture<a name="line.230"></a>
-<span class="sourceLineNo">231</span>        if (globalTracker &amp;&amp; trackerNode.isDeleted(procId) == DeleteState.YES) {<a name="line.231"></a>
-<span class="sourceLineNo">232</span>          node.delete(procId);<a name="line.232"></a>
-<span class="sourceLineNo">233</span>          continue;<a name="line.233"></a>
-<span class="sourceLineNo">234</span>        }<a name="line.234"></a>
-<span class="sourceLineNo">235</span>        if (trackerNode.isModified(procId)) {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>          // the procedure was modified<a name="line.236"></a>
-<span class="sourceLineNo">237</span>          node.delete(procId);<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        }<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      }<a name="line.239"></a>
-<span class="sourceLineNo">240</span>    }<a name="line.240"></a>
+<span class="sourceLineNo">164</span>    trackProcIds(procId);<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    return node;<a name="line.165"></a>
+<span class="sourceLineNo">166</span>  }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span>  /**<a name="line.168"></a>
+<span class="sourceLineNo">169</span>   * Will be called when restarting where we need to rebuild the ProcedureStoreTracker.<a name="line.169"></a>
+<span class="sourceLineNo">170</span>   */<a name="line.170"></a>
+<span class="sourceLineNo">171</span>  public void setMinMaxModifiedProcIds(long min, long max) {<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    this.minModifiedProcId = min;<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    this.maxModifiedProcId = max;<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  }<a name="line.174"></a>
+<span class="sourceLineNo">175</span>  /**<a name="line.175"></a>
+<span class="sourceLineNo">176</span>   * This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The<a name="line.176"></a>
+<span class="sourceLineNo">177</span>   * {@link #delete(long)} method above assume that the {@link BitSetNode} exists, but when restart<a name="line.177"></a>
+<span class="sourceLineNo">178</span>   * this is not true, as we will read the wal files in reverse order so a delete may come first.<a name="line.178"></a>
+<span class="sourceLineNo">179</span>   */<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  public void setDeleted(long procId, boolean isDeleted) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span>    BitSetNode node = getOrCreateNode(procId);<a name="line.181"></a>
+<span class="sourceLineNo">182</span>    assert node.contains(procId) : "expected procId=" + procId + " in the node=" + node;<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    node.updateState(procId, isDeleted);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>    trackProcIds(procId);<a name="line.184"></a>
+<span class="sourceLineNo">185</span>  }<a name="line.185"></a>
+<span class="sourceLineNo">186</span><a name="line.186"></a>
+<span class="sourceLineNo">187</span>  /**<a name="line.187"></a>
+<span class="sourceLineNo">188</span>   * Set the given bit for the procId to delete if it was modified before.<a name="line.188"></a>
+<span class="sourceLineNo">189</span>   * &lt;p/&gt;<a name="line.189"></a>
+<span class="sourceLineNo">190</span>   * This method is used to test whether a procedure wal file can be safely deleted, as if all the<a name="line.190"></a>
+<span class="sourceLineNo">191</span>   * procedures in the given procedure wal file has been modified in the new procedure wal files,<a name="line.191"></a>
+<span class="sourceLineNo">192</span>   * then we can delete it.<a name="line.192"></a>
+<span class="sourceLineNo">193</span>   */<a name="line.193"></a>
+<span class="sourceLineNo">194</span>  public void setDeletedIfModified(long... procId) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    BitSetNode node = null;<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    for (int i = 0; i &lt; procId.length; ++i) {<a name="line.196"></a>
+<span class="sourceLineNo">197</span>      node = lookupClosestNode(node, procId[i]);<a name="line.197"></a>
+<span class="sourceLineNo">198</span>      if (node != null &amp;&amp; node.isModified(procId[i])) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>        node.delete(procId[i]);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      }<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    }<a name="line.201"></a>
+<span class="sourceLineNo">202</span>  }<a name="line.202"></a>
+<span class="sourceLineNo">203</span><a name="line.203"></a>
+<span class="sourceLineNo">204</span>  private void setDeleteIf(ProcedureStoreTracker tracker,<a name="line.204"></a>
+<span class="sourceLineNo">205</span>      BiFunction&lt;BitSetNode, Long, Boolean&gt; func) {<a name="line.205"></a>
+<span class="sourceLineNo">206</span>    BitSetNode trackerNode = null;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    for (BitSetNode node : map.values()) {<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      long minProcId = node.getStart();<a name="line.208"></a>
+<span class="sourceLineNo">209</span>      long maxProcId = node.getEnd();<a name="line.209"></a>
+<span class="sourceLineNo">210</span>      for (long procId = minProcId; procId &lt;= maxProcId; ++procId) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>        if (!node.isModified(procId)) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span>          continue;<a name="line.212"></a>
+<span class="sourceLineNo">213</span>        }<a name="line.213"></a>
+<span class="sourceLineNo">214</span><a name="line.214"></a>
+<span class="sourceLineNo">215</span>        trackerNode = tracker.lookupClosestNode(trackerNode, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span>        if (func.apply(trackerNode, procId)) {<a name="line.216"></a>
+<span class="sourceLineNo">217</span>          node.delete(procId);<a name="line.217"></a>
+<span class="sourceLineNo">218</span>        }<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      }<a name="line.219"></a>
+<span class="sourceLineNo">220</span>    }<a name="line.220"></a>
+<span class="sourceLineNo">221</span>  }<a name="line.221"></a>
+<span class="sourceLineNo">222</span><a name="line.222"></a>
+<span class="sourceLineNo">223</span>  /**<a name="line.223"></a>
+<span class="sourceLineNo">224</span>   * For the global tracker, we will use this method to build the holdingCleanupTracker, as the<a name="line.224"></a>
+<span class="sourceLineNo">225</span>   * modified flags will be cleared after rolling so we only need to test the deleted flags.<a name="line.225"></a>
+<span class="sourceLineNo">226</span>   * @see #setDeletedIfModifiedInBoth(ProcedureStoreTracker)<a name="line.226"></a>
+<span class="sourceLineNo">227</span>   */<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  public void setDeletedIfDeletedByThem(ProcedureStoreTracker tracker) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span>    setDeleteIf(tracker, (node, procId) -&gt; node == null || !node.contains(procId) ||<a name="line.229"></a>
+<span class="sourceLineNo">230</span>      node.isDeleted(procId) == DeleteState.YES);<a name="line.230"></a>
+<span class="sourceLineNo">231</span>  }<a name="line.231"></a>
+<span class="sourceLineNo">232</span><a name="line.232"></a>
+<span class="sourceLineNo">233</span>  /**<a name="line.233"></a>
+<span class="sourceLineNo">234</span>   * Similar with {@link #setDeletedIfModified(long...)}, but here the {@code procId} are given by<a name="line.234"></a>
+<span class="sourceLineNo">235</span>   * the {@code tracker}. If a procedure is modified by us, and also by the given {@code tracker},<a name="line.235"></a>
+<span class="sourceLineNo">236</span>   * then we mark it as deleted.<a name="line.236"></a>
+<span class="sourceLineNo">237</span>   * @see #setDeletedIfModified(long...)<a name="line.237"></a>
+<span class="sourceLineNo">238</span>   */<a name="line.238"></a>
+<span class="sourceLineNo">239</span>  public void setDeletedIfModifiedInBoth(ProcedureStoreTracker tracker) {<a name="line.239"></a>
+<span class="sourceLineNo">240</span>    setDeleteIf(tracker, (node, procId) -&gt; node != null &amp;&amp; node.isModified(procId));<a name="line.240"></a>
 <span class="sourceLineNo">241</span>  }<a name="line.241"></a>
 <span class="sourceLineNo">242</span><a name="line.242"></a>
 <span class="sourceLineNo">243</span>  /**<a name="line.243"></a>


[41/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
index b7a2a38..6591d6c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.InitializationMonitor.html
@@ -1789,2174 +1789,2176 @@
 <span class="sourceLineNo">1781</span>      LOG.debug("Master has not been initialized, don't run region normalizer.");<a name="line.1781"></a>
 <span class="sourceLineNo">1782</span>      return false;<a name="line.1782"></a>
 <span class="sourceLineNo">1783</span>    }<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span><a name="line.1784"></a>
-<span class="sourceLineNo">1785</span>    if (isInMaintenanceMode()) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span>      return false;<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span>    }<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span><a name="line.1789"></a>
-<span class="sourceLineNo">1790</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1791"></a>
-<span class="sourceLineNo">1792</span>      return false;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span>    }<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span><a name="line.1794"></a>
-<span class="sourceLineNo">1795</span>    synchronized (this.normalizer) {<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span>      // Don't run the normalizer concurrently<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1797"></a>
-<span class="sourceLineNo">1798</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1798"></a>
-<span class="sourceLineNo">1799</span><a name="line.1799"></a>
-<span class="sourceLineNo">1800</span>      Collections.shuffle(allEnabledTables);<a name="line.1800"></a>
+<span class="sourceLineNo">1784</span>    if (this.getServerManager().isClusterShutdown()) {<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span>      LOG.info("Cluster is shutting down, don't run region normalizer.");<a name="line.1785"></a>
+<span class="sourceLineNo">1786</span>      return false;<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span>    }<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span>    if (isInMaintenanceMode()) {<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span>      return false;<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span>    }<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span>      return false;<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span>    }<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span><a name="line.1796"></a>
+<span class="sourceLineNo">1797</span>    synchronized (this.normalizer) {<a name="line.1797"></a>
+<span class="sourceLineNo">1798</span>      // Don't run the normalizer concurrently<a name="line.1798"></a>
+<span class="sourceLineNo">1799</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1800"></a>
 <span class="sourceLineNo">1801</span><a name="line.1801"></a>
-<span class="sourceLineNo">1802</span>      for (TableName table : allEnabledTables) {<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span>        if (isInMaintenanceMode()) {<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span>          return false;<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span>        }<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span><a name="line.1807"></a>
-<span class="sourceLineNo">1808</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span>          continue;<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span>        }<a name="line.1814"></a>
-<span class="sourceLineNo">1815</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span>        if (plans != null) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span>          for (NormalizationPlan plan : plans) {<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span>              splitPlanCount++;<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span>              mergePlanCount++;<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span>            }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span>          }<a name="line.1824"></a>
-<span class="sourceLineNo">1825</span>        }<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span>      }<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span>    }<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span>    // Return true indicating a success.<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span>    return true;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span>  }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span>  /**<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span>   */<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span>  @Override<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span>  public String getClientIdAuditPrefix() {<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span>  }<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span><a name="line.1841"></a>
-<span class="sourceLineNo">1842</span>  /**<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span>   * Switch for the background CatalogJanitor thread.<a name="line.1843"></a>
-<span class="sourceLineNo">1844</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span>   * if disabled.<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span>   */<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span>  }<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span><a name="line.1851"></a>
-<span class="sourceLineNo">1852</span>  @Override<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span>  public long mergeRegions(<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span>      final RegionInfo[] regionsToMerge,<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span>      final boolean forcible,<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span>      final long nonceGroup,<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span>      final long nonce) throws IOException {<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span>    checkInitialized();<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span><a name="line.1859"></a>
-<span class="sourceLineNo">1860</span>    assert(regionsToMerge.length == 2);<a name="line.1860"></a>
+<span class="sourceLineNo">1802</span>      Collections.shuffle(allEnabledTables);<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span><a name="line.1803"></a>
+<span class="sourceLineNo">1804</span>      for (TableName table : allEnabledTables) {<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span>        if (isInMaintenanceMode()) {<a name="line.1805"></a>
+<span class="sourceLineNo">1806</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span>          return false;<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span>        }<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span><a name="line.1809"></a>
+<span class="sourceLineNo">1810</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span>          continue;<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span>        }<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span>        if (plans != null) {<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span>          for (NormalizationPlan plan : plans) {<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1821"></a>
+<span class="sourceLineNo">1822</span>              splitPlanCount++;<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span>              mergePlanCount++;<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span>            }<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span>          }<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span>      }<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span>    }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span>    // Return true indicating a success.<a name="line.1831"></a>
+<span class="sourceLineNo">1832</span>    return true;<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span>  }<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span><a name="line.1834"></a>
+<span class="sourceLineNo">1835</span>  /**<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span>   */<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span>  @Override<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span>  public String getClientIdAuditPrefix() {<a name="line.1839"></a>
+<span class="sourceLineNo">1840</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span>  }<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span><a name="line.1843"></a>
+<span class="sourceLineNo">1844</span>  /**<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span>   * Switch for the background CatalogJanitor thread.<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span>   * if disabled.<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span>   */<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1850"></a>
+<span class="sourceLineNo">1851</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span>  }<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span><a name="line.1853"></a>
+<span class="sourceLineNo">1854</span>  @Override<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span>  public long mergeRegions(<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span>      final RegionInfo[] regionsToMerge,<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span>      final boolean forcible,<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span>      final long nonceGroup,<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span>      final long nonce) throws IOException {<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span>    checkInitialized();<a name="line.1860"></a>
 <span class="sourceLineNo">1861</span><a name="line.1861"></a>
-<span class="sourceLineNo">1862</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span>    }<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span><a name="line.1866"></a>
-<span class="sourceLineNo">1867</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span>      throw new IOException (<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1869"></a>
-<span class="sourceLineNo">1870</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span>    }<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span><a name="line.1872"></a>
-<span class="sourceLineNo">1873</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span>      throw new MergeRegionException(<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span>    }<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span><a name="line.1877"></a>
-<span class="sourceLineNo">1878</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1878"></a>
-<span class="sourceLineNo">1879</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span>      @Override<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span>      protected void run() throws IOException {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span><a name="line.1883"></a>
-<span class="sourceLineNo">1884</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span><a name="line.1886"></a>
-<span class="sourceLineNo">1887</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span>          regionsToMerge, forcible));<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span><a name="line.1889"></a>
-<span class="sourceLineNo">1890</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span>      }<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span><a name="line.1892"></a>
-<span class="sourceLineNo">1893</span>      @Override<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span>      protected String getDescription() {<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span>        return "MergeTableProcedure";<a name="line.1895"></a>
-<span class="sourceLineNo">1896</span>      }<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span>    });<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span>  }<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span>  @Override<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1901"></a>
-<span class="sourceLineNo">1902</span>      final long nonceGroup, final long nonce)<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span>  throws IOException {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span>    checkInitialized();<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1905"></a>
-<span class="sourceLineNo">1906</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>      @Override<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>      protected void run() throws IOException {<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span><a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>        // Execute the operation asynchronously<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span><a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>      @Override<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span>      protected String getDescription() {<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>        return "SplitTableProcedure";<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>      }<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>    });<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>  // Replace with an async implementation from which you can get<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span>  // a success/failure result.<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>  @VisibleForTesting<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span><a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    RegionInfo hri;<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>    if (regionState != null) {<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>      hri = regionState.getRegion();<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>    } else {<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span><a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>    ServerName dest;<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>        : new ArrayList&lt;&gt;(1);<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>      LOG.info(<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>              + " because the server is in exclude list");<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>      destServerName = null;<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>    }<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>        "choosing a server at random");<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>      exclude.add(regionState.getServerName());<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>      if (dest == null) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>        return;<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span>      }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    } else {<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span>      if (dest == null) {<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>        return;<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>      }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span>      // TODO: What is this? I don't get it.<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>        // regions on master.<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>          + " because it should not be on master");<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>        return;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span>      }<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    }<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span><a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>      return;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>    }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>    // Now we can do the move<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span><a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>    try {<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      checkInitialized();<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>      if (this.cpHost != null) {<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>      }<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span>      // closed<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span><a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>      try {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        // Is this going to work? Will we throw exception on error?<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        future.get();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>        throw new HBaseIOException(e);<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>      }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span>      if (this.cpHost != null) {<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>    } catch (IOException ioe) {<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      if (ioe instanceof HBaseIOException) {<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        throw (HBaseIOException)ioe;<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      }<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>      throw new HBaseIOException(ioe);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span>  }<a name="line.2014"></a>
-<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>  @Override<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>  public long createTable(<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>      final TableDescriptor tableDescriptor,<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>      final byte [][] splitKeys,<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>      final long nonceGroup,<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span>      final long nonce) throws IOException {<a name="line.2021"></a>
-<span class="sourceLineNo">2022</span>    checkInitialized();<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span><a name="line.2023"></a>
-<span class="sourceLineNo">2024</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span><a name="line.2026"></a>
-<span class="sourceLineNo">2027</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2027"></a>
-<span class="sourceLineNo">2028</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2028"></a>
-<span class="sourceLineNo">2029</span><a name="line.2029"></a>
-<span class="sourceLineNo">2030</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span>      @Override<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span>      protected void run() throws IOException {<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2034"></a>
-<span class="sourceLineNo">2035</span><a name="line.2035"></a>
-<span class="sourceLineNo">2036</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2036"></a>
+<span class="sourceLineNo">1862</span>    assert(regionsToMerge.length == 2);<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span><a name="line.1863"></a>
+<span class="sourceLineNo">1864</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span><a name="line.1868"></a>
+<span class="sourceLineNo">1869</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span>      throw new IOException (<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span>    }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span><a name="line.1874"></a>
+<span class="sourceLineNo">1875</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span>      throw new MergeRegionException(<a name="line.1876"></a>
+<span class="sourceLineNo">1877</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span>    }<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span><a name="line.1879"></a>
+<span class="sourceLineNo">1880</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span>      @Override<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span>      protected void run() throws IOException {<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span><a name="line.1888"></a>
+<span class="sourceLineNo">1889</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span>          regionsToMerge, forcible));<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span><a name="line.1891"></a>
+<span class="sourceLineNo">1892</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span>      }<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span><a name="line.1894"></a>
+<span class="sourceLineNo">1895</span>      @Override<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span>      protected String getDescription() {<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span>        return "MergeTableProcedure";<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span>      }<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span>    });<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span>  }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span><a name="line.1901"></a>
+<span class="sourceLineNo">1902</span>  @Override<a name="line.1902"></a>
+<span class="sourceLineNo">1903</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span>      final long nonceGroup, final long nonce)<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span>  throws IOException {<a name="line.1905"></a>
+<span class="sourceLineNo">1906</span>    checkInitialized();<a name="line.1906"></a>
+<span class="sourceLineNo">1907</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>      @Override<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>      protected void run() throws IOException {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        // Execute the operation asynchronously<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>      }<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>      @Override<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>      protected String getDescription() {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span>        return "SplitTableProcedure";<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>      }<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>    });<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  }<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span><a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span>  // Replace with an async implementation from which you can get<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>  // a success/failure result.<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span>  @VisibleForTesting<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    RegionInfo hri;<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    if (regionState != null) {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      hri = regionState.getRegion();<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    } else {<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span>    }<a name="line.1938"></a>
+<span class="sourceLineNo">1939</span><a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>    ServerName dest;<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>        : new ArrayList&lt;&gt;(1);<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>      LOG.info(<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span>              + " because the server is in exclude list");<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>      destServerName = null;<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    }<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span>        "choosing a server at random");<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>      exclude.add(regionState.getServerName());<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span>      if (dest == null) {<a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        return;<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>      }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span>    } else {<a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>      if (dest == null) {<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>        return;<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      }<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>      // TODO: What is this? I don't get it.<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>        // regions on master.<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>          + " because it should not be on master");<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>        return;<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>      }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span>    }<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span><a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>      return;<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>    }<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span><a name="line.1983"></a>
+<span class="sourceLineNo">1984</span>    // Now we can do the move<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span><a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>    try {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      checkInitialized();<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      if (this.cpHost != null) {<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span>      }<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      // closed<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span><a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      try {<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>        // Is this going to work? Will we throw exception on error?<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        future.get();<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        throw new HBaseIOException(e);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>      if (this.cpHost != null) {<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>      }<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>    } catch (IOException ioe) {<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>      if (ioe instanceof HBaseIOException) {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>        throw (HBaseIOException)ioe;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>      }<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      throw new HBaseIOException(ioe);<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>    }<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>  }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>  @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>  public long createTable(<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      final TableDescriptor tableDescriptor,<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>      final byte [][] splitKeys,<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>      final long nonceGroup,<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>      final long nonce) throws IOException {<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span>    checkInitialized();<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span><a name="line.2025"></a>
+<span class="sourceLineNo">2026</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2027"></a>
+<span class="sourceLineNo">2028</span><a name="line.2028"></a>
+<span class="sourceLineNo">2029</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span><a name="line.2031"></a>
+<span class="sourceLineNo">2032</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span>      @Override<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span>      protected void run() throws IOException {<a name="line.2035"></a>
+<span class="sourceLineNo">2036</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2036"></a>
 <span class="sourceLineNo">2037</span><a name="line.2037"></a>
-<span class="sourceLineNo">2038</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span>        //<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2042"></a>
-<span class="sourceLineNo">2043</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span>        submitProcedure(new CreateTableProcedure(<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span>        latch.await();<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span>      }<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span><a name="line.2050"></a>
-<span class="sourceLineNo">2051</span>      @Override<a name="line.2051"></a>
-<span class="sourceLineNo">2052</span>      protected String getDescription() {<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span>        return "CreateTableProcedure";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span>      }<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span>    });<a name="line.2055"></a>
-<span class="sourceLineNo">2056</span>  }<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span><a name="line.2057"></a>
-<span class="sourceLineNo">2058</span>  @Override<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2059"></a>
-<span class="sourceLineNo">2060</span>    if (isStopped()) {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span>      throw new MasterNotRunningException();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span>    }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span>    if (!(tableName.isSystemTable())) {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span>      throw new IllegalArgumentException(<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span>        "Only system table creation can use this createSystemTable API");<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span>    }<a name="line.2068"></a>
-<span class="sourceLineNo">2069</span><a name="line.2069"></a>
-<span class="sourceLineNo">2070</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2070"></a>
+<span class="sourceLineNo">2038</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span><a name="line.2039"></a>
+<span class="sourceLineNo">2040</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2041"></a>
+<span class="sourceLineNo">2042</span>        //<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span>        submitProcedure(new CreateTableProcedure(<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span>        latch.await();<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span>      }<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span><a name="line.2052"></a>
+<span class="sourceLineNo">2053</span>      @Override<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span>      protected String getDescription() {<a name="line.2054"></a>
+<span class="sourceLineNo">2055</span>        return "CreateTableProcedure";<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span>      }<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span>    });<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span>  }<a name="line.2058"></a>
+<span class="sourceLineNo">2059</span><a name="line.2059"></a>
+<span class="sourceLineNo">2060</span>  @Override<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span>    if (isStopped()) {<a name="line.2062"></a>
+<span class="sourceLineNo">2063</span>      throw new MasterNotRunningException();<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span>    }<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span><a name="line.2065"></a>
+<span class="sourceLineNo">2066</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2066"></a>
+<span class="sourceLineNo">2067</span>    if (!(tableName.isSystemTable())) {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span>      throw new IllegalArgumentException(<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span>        "Only system table creation can use this createSystemTable API");<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span>    }<a name="line.2070"></a>
 <span class="sourceLineNo">2071</span><a name="line.2071"></a>
-<span class="sourceLineNo">2072</span>    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2072"></a>
+<span class="sourceLineNo">2072</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2072"></a>
 <span class="sourceLineNo">2073</span><a name="line.2073"></a>
-<span class="sourceLineNo">2074</span>    // This special create table is called locally to master.  Therefore, no RPC means no need<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span>    // to use nonce to detect duplicated RPC call.<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span>    long procId = this.procedureExecutor.submitProcedure(<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span>      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span><a name="line.2078"></a>
-<span class="sourceLineNo">2079</span>    return procId;<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span>  }<a name="line.2080"></a>
-<span class="sourceLineNo">2081</span><a name="line.2081"></a>
-<span class="sourceLineNo">2082</span>  /**<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span>   * Checks whether the table conforms to some sane limits, and configured<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span>   * values (compression, etc) work. Throws an exception if something is wrong.<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span>   * @throws IOException<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span>   */<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span>  private void sanityCheckTableDescriptor(final TableDescriptor htd) throws IOException {<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span>    final String CONF_KEY = "hbase.table.sanity.checks";<a name="line.2088"></a>
-<span class="sourceLineNo">2089</span>    boolean logWarn = false;<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span>    if (!conf.getBoolean(CONF_KEY, true)) {<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span>      logWarn = true;<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span>    }<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span>    String tableVal = htd.getValue(CONF_KEY);<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span>    if (tableVal != null &amp;&amp; !Boolean.valueOf(tableVal)) {<a name="line.2094"></a>
-<span class="sourceLineNo">2095</span>      logWarn = true;<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span>    }<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span><a name="line.2097"></a>
-<span class="sourceLineNo">2098</span>    // check max file size<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span>    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span>    long maxFileSize = htd.getMaxFileSize();<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span>    if (maxFileSize &lt; 0) {<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span>      maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span>    }<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span>    if (maxFileSize &lt; conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span>      String message = "MAX_FILESIZE for table descriptor or "<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span>          + "\"hbase.hregion.max.filesize\" (" + maxFileSize<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span>          + ") is too small, which might cause over splitting into unmanageable "<a name="line.2107"></a>
-<span class="sourceLineNo">2108</span>          + "number of regions.";<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span>    }<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span><a name="line.2111"></a>
-<span class="sourceLineNo">2112</span>    // check flush size<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span>    long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span>    long flushSize = htd.getMemStoreFlushSize();<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span>    if (flushSize &lt; 0) {<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span>      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);<a name="line.2116"></a>
-<span class="sourceLineNo">2117</span>    }<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span>    if (flushSize &lt; conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span>      String message = "MEMSTORE_FLUSHSIZE for table descriptor or "<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span>          + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span>          + " very frequent flushing.";<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2122"></a>
-<span class="sourceLineNo">2123</span>    }<a name="line.2123"></a>
-<span class="sourceLineNo">2124</span><a name="line.2124"></a>
-<span class="sourceLineNo">2125</span>    // check that coprocessors and other specified plugin classes can be loaded<a name="line.2125"></a>
-<span class="sourceLineNo">2126</span>    try {<a name="line.2126"></a>
-<span class="sourceLineNo">2127</span>      checkClassLoading(conf, htd);<a name="line.2127"></a>
-<span class="sourceLineNo">2128</span>    } catch (Exception ex) {<a name="line.2128"></a>
-<span class="sourceLineNo">2129</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);<a name="line.2129"></a>
-<span class="sourceLineNo">2130</span>    }<a name="line.2130"></a>
-<span class="sourceLineNo">2131</span><a name="line.2131"></a>
-<span class="sourceLineNo">2132</span>    // check compression can be loaded<a name="line.2132"></a>
-<span class="sourceLineNo">2133</span>    try {<a name="line.2133"></a>
-<span class="sourceLineNo">2134</span>      checkCompression(htd);<a name="line.2134"></a>
-<span class="sourceLineNo">2135</span>    } catch (IOException e) {<a name="line.2135"></a>
-<span class="sourceLineNo">2136</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2136"></a>
-<span class="sourceLineNo">2137</span>    }<a name="line.2137"></a>
-<span class="sourceLineNo">2138</span><a name="line.2138"></a>
-<span class="sourceLineNo">2139</span>    // check encryption can be loaded<a name="line.2139"></a>
-<span class="sourceLineNo">2140</span>    try {<a name="line.2140"></a>
-<span class="sourceLineNo">2141</span>      checkEncryption(conf, htd);<a name="line.2141"></a>
-<span class="sourceLineNo">2142</span>    } catch (IOException e) {<a name="line.2142"></a>
-<span class="sourceLineNo">2143</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2143"></a>
-<span class="sourceLineNo">2144</span>    }<a name="line.2144"></a>
-<span class="sourceLineNo">2145</span>    // Verify compaction policy<a name="line.2145"></a>
-<span class="sourceLineNo">2146</span>    try{<a name="line.2146"></a>
-<span class="sourceLineNo">2147</span>      checkCompactionPolicy(conf, htd);<a name="line.2147"></a>
-<span class="sourceLineNo">2148</span>    } catch(IOException e){<a name="line.2148"></a>
-<span class="sourceLineNo">2149</span>      warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);<a name="line.2149"></a>
-<span class="sourceLineNo">2150</span>    }<a name="line.2150"></a>
-<span class="sourceLineNo">2151</span>    // check that we have at least 1 CF<a name="line.2151"></a>
-<span class="sourceLineNo">2152</span>    if (htd.getColumnFamilyCount() == 0) {<a name="line.2152"></a>
-<span class="sourceLineNo">2153</span>      String message = "Table should have at least one column family.";<a name="line.2153"></a>
-<span class="sourceLineNo">2154</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2154"></a>
-<span class="sourceLineNo">2155</span>    }<a name="line.2155"></a>
-<span class="sourceLineNo">2156</span><a name="line.2156"></a>
-<span class="sourceLineNo">2157</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2157"></a>
-<span class="sourceLineNo">2158</span>      if (hcd.getTimeToLive() &lt;= 0) {<a name="line.2158"></a>
-<span class="sourceLineNo">2159</span>        String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";<a name="line.2159"></a>
-<span class="sourceLineNo">2160</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2160"></a>
-<span class="sourceLineNo">2161</span>      }<a name="line.2161"></a>
-<span class="sourceLineNo">2162</span><a name="line.2162"></a>
-<span class="sourceLineNo">2163</span>      // check blockSize<a name="line.2163"></a>
-<span class="sourceLineNo">2164</span>      if (hcd.getBlocksize() &lt; 1024 || hcd.getBlocksize() &gt; 16 * 1024 * 1024) {<a name="line.2164"></a>
-<span class="sourceLineNo">2165</span>        String message = "Block size for column family " + hcd.getNameAsString()<a name="line.2165"></a>
-<span class="sourceLineNo">2166</span>            + "  must be between 1K and 16MB.";<a name="line.2166"></a>
-<span class="sourceLineNo">2167</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2167"></a>
-<span class="sourceLineNo">2168</span>      }<a name="line.2168"></a>
-<span class="sourceLineNo">2169</span><a name="line.2169"></a>
-<span class="sourceLineNo">2170</span>      // check versions<a name="line.2170"></a>
-<span class="sourceLineNo">2171</span>      if (hcd.getMinVersions() &lt; 0) {<a name="line.2171"></a>
-<span class="sourceLineNo">2172</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2172"></a>
-<span class="sourceLineNo">2173</span>          + "  must be positive.";<a name="line.2173"></a>
-<span class="sourceLineNo">2174</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2174"></a>
-<span class="sourceLineNo">2175</span>      }<a name="line.2175"></a>
-<span class="sourceLineNo">2176</span>      // max versions already being checked<a name="line.2176"></a>
-<span class="sourceLineNo">2177</span><a name="line.2177"></a>
-<span class="sourceLineNo">2178</span>      // HBASE-13776 Setting illegal versions for ColumnFamilyDescriptor<a name="line.2178"></a>
-<span class="sourceLineNo">2179</span>      //  does not throw IllegalArgumentException<a name="line.2179"></a>
-<span class="sourceLineNo">2180</span>      // check minVersions &lt;= maxVerions<a name="line.2180"></a>
-<span class="sourceLineNo">2181</span>      if (hcd.getMinVersions() &gt; hcd.getMaxVersions()) {<a name="line.2181"></a>
-<span class="sourceLineNo">2182</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2182"></a>
-<span class="sourceLineNo">2183</span>            + " must be less than the Max versions.";<a name="line.2183"></a>
-<span class="sourceLineNo">2184</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2184"></a>
-<span class="sourceLineNo">2185</span>      }<a name="line.2185"></a>
-<span class="sourceLineNo">2186</span><a name="line.2186"></a>
-<span class="sourceLineNo">2187</span>      // check replication scope<a name="line.2187"></a>
-<span class="sourceLineNo">2188</span>      checkReplicationScope(hcd);<a name="line.2188"></a>
-<span class="sourceLineNo">2189</span>      // check bloom filter type<a name="line.2189"></a>
-<span class="sourceLineNo">2190</span>      checkBloomFilterType(hcd);<a name="line.2190"></a>
-<span class="sourceLineNo">2191</span><a name="line.2191"></a>
-<span class="sourceLineNo">2192</span>      // check data replication factor, it can be 0(default value) when user has not explicitly<a name="line.2192"></a>
-<span class="sourceLineNo">2193</span>      // set the value, in this case we use default replication factor set in the file system.<a name="line.2193"></a>
-<span class="sourceLineNo">2194</span>      if (hcd.getDFSReplication() &lt; 0) {<a name="line.2194"></a>
-<span class="sourceLineNo">2195</span>        String message = "HFile Replication for column family " + hcd.getNameAsString()<a name="line.2195"></a>
-<span class="sourceLineNo">2196</span>            + "  must be greater than zero.";<a name="line.2196"></a>
-<span class="sourceLineNo">2197</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2197"></a>
-<span class="sourceLineNo">2198</span>      }<a name="line.2198"></a>
-<span class="sourceLineNo">2199</span><a name="line.2199"></a>
-<span class="sourceLineNo">2200</span>      // TODO: should we check coprocessors and encryption ?<a name="line.2200"></a>
-<span class="sourceLineNo">2201</span>    }<a name="line.2201"></a>
-<span class="sourceLineNo">2202</span>  }<a name="line.2202"></a>
-<span class="sourceLineNo">2203</span><a name="line.2203"></a>
-<span class="sourceLineNo">2204</span>  private void checkReplicationScope(ColumnFamilyDescriptor hcd) throws IOException{<a name="line.2204"></a>
-<span class="sourceLineNo">2205</span>    // check replication scope<a name="line.2205"></a>
-<span class="sourceLineNo">2206</span>    WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(hcd.getScope());<a name="line.2206"></a>
-<span class="sourceLineNo">2207</span>    if (scop == null) {<a name="line.2207"></a>
-<span class="sourceLineNo">2208</span>      String message = "Replication scope for column family "<a name="line.2208"></a>
-<span class="sourceLineNo">2209</span>          + hcd.getNameAsString() + " is " + hcd.getScope() + " which is invalid.";<a name="line.2209"></a>
-<span class="sourceLineNo">2210</span><a name="line.2210"></a>
-<span class="sourceLineNo">2211</span>      LOG.error(message);<a name="line.2211"></a>
-<span class="sourceLineNo">2212</span>      throw new DoNotRetryIOException(message);<a name="line.2212"></a>
-<span class="sourceLineNo">2213</span>    }<a name="line.2213"></a>
-<span class="sourceLineNo">2214</span>  }<a name="line.2214"></a>
-<span class="sourceLineNo">2215</span><a name="line.2215"></a>
-<span class="sourceLineNo">2216</span>  private void checkCompactionPolicy(Configuration conf, TableDescriptor htd)<a name="line.2216"></a>
-<span class="sourceLineNo">2217</span>      throws IOException {<a name="line.2217"></a>
-<span class="sourceLineNo">2218</span>    // FIFO compaction has some requirements<a name="line.2218"></a>
-<span class="sourceLineNo">2219</span>    // Actually FCP ignores periodic major compactions<a name="line.2219"></a>
-<span class="sourceLineNo">2220</span>    String className = htd.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2220"></a>
-<span class="sourceLineNo">2221</span>    if (className == null) {<a name="line.2221"></a>
-<span class="sourceLineNo">2222</span>      className =<a name="line.2222"></a>
-<span class="sourceLineNo">2223</span>          conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,<a name="line.2223"></a>
-<span class="sourceLineNo">2224</span>            ExploringCompactionPolicy.class.getName());<a name="line.2224"></a>
-<span class="sourceLineNo">2225</span>    }<a name="line.2225"></a>
-<span class="sourceLineNo">2226</span><a name="line.2226"></a>
-<span class="sourceLineNo">2227</span>    int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;<a name="line.2227"></a>
-<span class="sourceLineNo">2228</span>    String sv = htd.getValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2228"></a>
-<span class="sourceLineNo">2229</span>    if (sv != null) {<a name="line.2229"></a>
-<span class="sourceLineNo">2230</span>      blockingFileCount = Integer.parseInt(sv);<a name="line.2230"></a>
-<span class="sourceLineNo">2231</span>    } else {<a name="line.2231"></a>
-<span class="sourceLineNo">2232</span>      blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);<a name="line.2232"></a>
-<span class="sourceLineNo">2233</span>    }<a name="line.2233"></a>
-<span class="sourceLineNo">2234</span><a name="line.2234"></a>
-<span class="sourceLineNo">2235</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2235"></a>
-<span class="sourceLineNo">2236</span>      String compactionPolicy =<a name="line.2236"></a>
-<span class="sourceLineNo">2237</span>          hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2237"></a>
-<span class="sourceLineNo">2238</span>      if (compactionPolicy == null) {<a name="line.2238"></a>
-<span class="sourceLineNo">2239</span>        compactionPolicy = className;<a name="line.2239"></a>
-<span class="sourceLineNo">2240</span>      }<a name="line.2240"></a>
-<span class="sourceLineNo">2241</span>      if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {<a name="line.2241"></a>
-<span class="sourceLineNo">2242</span>        continue;<a name="line.2242"></a>
-<span class="sourceLineNo">2243</span>      }<a name="line.2243"></a>
-<span class="sourceLineNo">2244</span>      // FIFOCompaction<a name="line.2244"></a>
-<span class="sourceLineNo">2245</span>      String message = null;<a name="line.2245"></a>
-<span class="sourceLineNo">2246</span><a name="line.2246"></a>
-<span class="sourceLineNo">2247</span>      // 1. Check TTL<a name="line.2247"></a>
-<span class="sourceLineNo">2248</span>      if (hcd.getTimeToLive() == ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {<a name="line.2248"></a>
-<span class="sourceLineNo">2249</span>        message = "Default TTL is not supported for FIFO compaction";<a name="line.2249"></a>
-<span class="sourceLineNo">2250</span>        throw new IOException(message);<a name="line.2250"></a>
-<span class="sourceLineNo">2251</span>      }<a name="line.2251"></a>
-<span class="sourceLineNo">2252</span><a name="line.2252"></a>
-<span class="sourceLineNo">2253</span>      // 2. Check min versions<a name="line.2253"></a>
-<span class="sourceLineNo">2254</span>      if (hcd.getMinVersions() &gt; 0) {<a name="line.2254"></a>
-<span class="sourceLineNo">2255</span>        message = "MIN_VERSION &gt; 0 is not supported for FIFO compaction";<a name="line.2255"></a>
-<span class="sourceLineNo">2256</span>        throw new IOException(message);<a name="line.2256"></a>
-<span class="sourceLineNo">2257</span>      }<a name="line.2257"></a>
-<span class="sourceLineNo">2258</span><a name="line.2258"></a>
-<span class="sourceLineNo">2259</span>      // 3. blocking file count<a name="line.2259"></a>
-<span class="sourceLineNo">2260</span>      sv = hcd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2260"></a>
-<span class="sourceLineNo">2261</span>      if (sv != null) {<a name="line.2261"></a>
-<span class="sourceLineNo">2262</span>        blockingFileCount = Integer.parseInt(sv);<a name="line.2262"></a>
-<span class="sourceLineNo">2263</span>      }<a name="line.2263"></a>
-<span class="sourceLineNo">2264</span>      if (blockingFileCount &lt; 1000) {<a name="line.2264"></a>
-<span class="sourceLineNo">2265</span>        message =<a name="line.2265"></a>
-<span class="sourceLineNo">2266</span>            "Blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount<a name="line.2266"></a>
-<span class="sourceLineNo">2267</span>                + " is below recommended minimum of 1000 for column family "+ hcd.getNameAsString();<a name="line.2267"></a>
-<span class="sourceLineNo">2268</span>        throw new IOException(message);<a name="line.2268"></a>
-<span class="sourceLineNo">2269</span>      }<a name="line.2269"></a>
-<span class="sourceLineNo">2270</span>    }<a name="line.2270"></a>
-<span class="sourceLineNo">2271</span>  }<a name="line.2271"></a>
-<span class="sourceLineNo">2272</span><a name="line.2272"></a>
-<span class="sourceLineNo">2273</span>  private static void checkBloomFilterType(ColumnFamilyDescriptor cfd)<a name="line.2273"></a>
-<span class="sourceLineNo">2274</span>      throws IOException {<a name="line.2274"></a>
-<span class="sourceLineNo">2275</span>    Configuration conf = new CompoundConfiguration().addStringMap(cfd.getConfiguration());<a name="line.2275"></a>
-<span class="sourceLineNo">2276</span>    try {<a name="line.2276"></a>
-<span class="sourceLineNo">2277</span>      BloomFilterUtil.getBloomFilterParam(cfd.getBloomFilterType(), conf);<a name="line.2277"></a>
-<span class="sourceLineNo">2278</span>    } catch (IllegalArgumentException e) {<a name="line.2278"></a>
-<span class="sourceLineNo">2279</span>      throw new DoNotRetryIOException("Failed to get bloom filter param", e);<a name="line.2279"></a>
-<span class="sourceLineNo">2280</span>    }<a name="line.2280"></a>
-<span class="sourceLineNo">2281</span>  }<a name="line.2281"></a>
-<span class="sourceLineNo">2282</span><a name="line.2282"></a>
-<span class="sourceLineNo">2283</span>  // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.<a name="line.2283"></a>
-<span class="sourceLineNo">2284</span>  private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,<a name="line.2284"></a>
-<span class="sourceLineNo">2285</span>      String message, Exception cause) throws IOException {<a name="line.2285"></a>
-<span class="sourceLineNo">2286</span>    if (!logWarn) {<a name="line.2286"></a>
-<span class="sourceLineNo">2287</span>      throw new DoNotRetryIOException(message + " Set " + confKey +<a name="line.2287"></a>
-<span class="sourceLineNo">2288</span>          " to false at conf or table descriptor if you want to bypass sanity checks", cause);<a name="line.2288"></a>
-<span class="sourceLineNo">2289</span>    }<a name="line.2289"></a>
-<span class="sourceLineNo">2290</span>    LOG.warn(message);<a name="line.2290"></a>
-<span class="sourceLineNo">2291</span>  }<a name="line.2291"></a>
-<span class="sourceLineNo">2292</span><a name="line.2292"></a>
-<span class="sourceLineNo">2293</span>  private void startActiveMasterManager(int infoPort) throws KeeperException {<a name="line.2293"></a>
-<span class="sourceLineNo">2294</span>    String backupZNode = ZNodePaths.joinZNode(<a name="line.2294"></a>
-<span class="sourceLineNo">2295</span>      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());<a name="line.2295"></a>
-<span class="sourceLineNo">2296</span>    /*<a name="line.2296"></a>
-<span class="sourceLineNo">2297</span>    * Add a ZNode for ourselves in the backup master directory since we<a name="line.2297"></a>
-<span class="sourceLineNo">2298</span>    * may not become the active master. If so, we want the actual active<a name="line.2298"></a>
-<span class="sourceLineNo">2299</span>    * master to know we are backup masters, so that it won't assign<a name="line.2299"></a>
-<span class="sourceLineNo">2300</span>    * regions to us if so configured.<a name="line.2300"></a>
-<span class="sourceLineNo">2301</span>    *<a name="line.2301"></a>
-<span class="sourceLineNo">2302</span>    * If we become the active master later, ActiveMasterManager will delete<a name="line.2302"></a>
-<span class="sourceLineNo">2303</span>    * this node explicitly.  If we crash before then, ZooKeeper will delete<a name="line.2303"></a>
-<span class="sourceLineNo">2304</span>    * this node for us since it is ephemeral.<a name="line.2304"></a>
-<span class="sourceLineNo">2305</span>    */<a name="line.2305"></a>
-<span class="sourceLineNo">2306</span>    LOG.info("Adding backup master ZNode " + backupZNode);<a name="line.2306"></a>
-<span class="sourceLineNo">2307</span>    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {<a name="line.2307"></a>
-<span class="sourceLineNo">2308</span>      LOG.warn("Failed create of " + backupZNode + " by " + serverName);<a name="line.2308"></a>
-<span class="sourceLineNo">2309</span>    }<a name="line.2309"></a>
-<span class="sourceLineNo">2310</span>    this.activeMasterManager.setInfoPort(infoPort);<a name="line.2310"></a>
-<span class="sourceLineNo">2311</span>    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);<a name="line.2311"></a>
-<span class="sourceLineNo">2312</span>    // If we're a backup master, stall until a primary to write this address<a name="line.2312"></a>
-<span class="sourceLineNo">2313</span>    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {<a name="line.2313"></a>
-<span class="sourceLineNo">2314</span>      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");<a name="line.2314"></a>
-<span class="sourceLineNo">2315</span>      // This will only be a minute or so while the cluster starts up,<a name="line.2315"></a>
-<span class="sourceLineNo">2316</span>      // so don't worry about setting watches on the parent znode<a name="line.2316"></a>
-<span class="sourceLineNo">2317</span>      while (!activeMasterManager.hasActiveMaster()) {<a name="line.2317"></a>
-<span class="sourceLineNo">2318</span>        LOG.debug("Waiting for master address and cluster state znode to be written.");<a name="line.2318"></a>
-<span class="sourceLineNo">2319</span>        Threads.sleep(timeout);<a name="line.2319"></a>
-<span class="sourceLineNo">2320</span>      }<a name="line.2320"></a>
-<span class="sourceLineNo">2321</span>    }<a name="line.2321"></a>
-<span class="sourceLineNo">2322</span>    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");<a name="line.2322"></a>
-<span class="sourceLineNo">2323</span>    status.setDescription("Master startup");<a name="line.2323"></a>
-<span class="sourceLineNo">2324</span>    try {<a name="line.2324"></a>
-<span class="sourceLineNo">2325</span>      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {<a name="line.2325"></a>
-<span class="sourceLineNo">2326</span>        finishActiveMasterInitialization(status);<a name="line.2326"></a>
-<span class="sourceLineNo">2327</span>      }<a name="line.2327"></a>
-<span class="sourceLineNo">2328</span>    } catch (Throwable t) {<a name="line.2328"></a>
-<span class="sourceLineNo">2329</span>      status.setStatus("Failed to become active: " + t.getMessage());<a name="line.2329"></a>
-<span class="sourceLineNo">2330</span>      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);<a name="line.2330"></a>
-<span class="sourceLineNo">2331</span>      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility<a name="line.2331"></a>
-<span class="sourceLineNo">2332</span>      if (t instanceof NoClassDefFoundError &amp;&amp; t.getMessage().<a name="line.2332"></a>
-<span class="sourceLineNo">2333</span>          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {<a name="line.2333"></a>
-<span class="sourceLineNo">2334</span>        // improved error message for this special case<a name="line.2334"></a>
-<span class="sourceLineNo">2335</span>        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +<a name="line.2335"></a>
-<span class="sourceLineNo">2336</span>          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +<a name="line.2336"></a>
-<span class="sourceLineNo">2337</span>          " or change your hadoop jars to start properly", t);<a name="line.2337"></a>
-<span class="sourceLineNo">2338</span>      } else {<a name="line.2338"></a>
-<span class="sourceLineNo">2339</span>        abort("Unhandled exception. Starting shutdown.", t);<a name="line.2339"></a>
-<span class="sourceLineNo">2340</span>      }<a name="line.2340"></a>
-<span class="sourceLineNo">2341</span>    } finally {<a name="line.2341"></a>
-<span class="sourceLineNo">2342</span>      status.cleanup();<a name="line.2342"></a>
-<span class="sourceLineNo">2343</span>    }<a name="line.2343"></a>
-<span class="sourceLineNo">2344</span>  }<a name="line.2344"></a>
-<span class="sourceLineNo">2345</span><a name="line.2345"></a>
-<span class="sourceLineNo">2346</span>  private void checkCompression(final TableDescriptor htd)<a name="line.2346"></a>
-<span class="sourceLineNo">2347</span>  throws IOException {<a name="line.2347"></a>
-<span class="sourceLineNo">2348</span>    if (!this.masterCheckCompression) return;<a name="line.2348"></a>
-<span class="sourceLineNo">2349</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2349"></a>
-<span class="sourceLineNo">2350</span>      checkCompression(hcd);<a name="line.2350"></a>
-<span class="sourceLineNo">2351</span>    }<a name="line.2351"></a>
-<span class="sourceLineNo">2352</span>  }<a name="line.2352"></a>
-<span class="sourceLineNo">2353</span><a name="line.2353"></a>
-<span class="sourceLineNo">2354</span>  private void checkCompression(final ColumnFamilyDescriptor hcd)<a name="line.2354"></a>
-<span class="sourceLineNo">2355</span>  throws IOException {<a name="line.2355"></a>
-<span class="sourceLineNo">2356</span>    if (!this.masterCheckCompression) return;<a name="line.2356"></a>
-<span class="sourceLineNo">2357</span>    CompressionTest.testCompression(hcd.getCompressionType());<a name="line.2357"></a>
-<span class="sourceLineNo">2358</span>    CompressionTest.testCompression(hcd.getCompactionCompressionType());<a name="line.2358"><

<TRUNCATED>

[40/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
index b7a2a38..6591d6c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.MasterStoppedException.html
@@ -1789,2174 +1789,2176 @@
 <span class="sourceLineNo">1781</span>      LOG.debug("Master has not been initialized, don't run region normalizer.");<a name="line.1781"></a>
 <span class="sourceLineNo">1782</span>      return false;<a name="line.1782"></a>
 <span class="sourceLineNo">1783</span>    }<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span><a name="line.1784"></a>
-<span class="sourceLineNo">1785</span>    if (isInMaintenanceMode()) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span>      return false;<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span>    }<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span><a name="line.1789"></a>
-<span class="sourceLineNo">1790</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1791"></a>
-<span class="sourceLineNo">1792</span>      return false;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span>    }<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span><a name="line.1794"></a>
-<span class="sourceLineNo">1795</span>    synchronized (this.normalizer) {<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span>      // Don't run the normalizer concurrently<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1797"></a>
-<span class="sourceLineNo">1798</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1798"></a>
-<span class="sourceLineNo">1799</span><a name="line.1799"></a>
-<span class="sourceLineNo">1800</span>      Collections.shuffle(allEnabledTables);<a name="line.1800"></a>
+<span class="sourceLineNo">1784</span>    if (this.getServerManager().isClusterShutdown()) {<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span>      LOG.info("Cluster is shutting down, don't run region normalizer.");<a name="line.1785"></a>
+<span class="sourceLineNo">1786</span>      return false;<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span>    }<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span>    if (isInMaintenanceMode()) {<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span>      return false;<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span>    }<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span>      return false;<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span>    }<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span><a name="line.1796"></a>
+<span class="sourceLineNo">1797</span>    synchronized (this.normalizer) {<a name="line.1797"></a>
+<span class="sourceLineNo">1798</span>      // Don't run the normalizer concurrently<a name="line.1798"></a>
+<span class="sourceLineNo">1799</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1800"></a>
 <span class="sourceLineNo">1801</span><a name="line.1801"></a>
-<span class="sourceLineNo">1802</span>      for (TableName table : allEnabledTables) {<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span>        if (isInMaintenanceMode()) {<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span>          return false;<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span>        }<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span><a name="line.1807"></a>
-<span class="sourceLineNo">1808</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span>          continue;<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span>        }<a name="line.1814"></a>
-<span class="sourceLineNo">1815</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span>        if (plans != null) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span>          for (NormalizationPlan plan : plans) {<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span>              splitPlanCount++;<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span>              mergePlanCount++;<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span>            }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span>          }<a name="line.1824"></a>
-<span class="sourceLineNo">1825</span>        }<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span>      }<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span>    }<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span>    // Return true indicating a success.<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span>    return true;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span>  }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span>  /**<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span>   */<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span>  @Override<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span>  public String getClientIdAuditPrefix() {<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span>  }<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span><a name="line.1841"></a>
-<span class="sourceLineNo">1842</span>  /**<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span>   * Switch for the background CatalogJanitor thread.<a name="line.1843"></a>
-<span class="sourceLineNo">1844</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span>   * if disabled.<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span>   */<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span>  }<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span><a name="line.1851"></a>
-<span class="sourceLineNo">1852</span>  @Override<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span>  public long mergeRegions(<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span>      final RegionInfo[] regionsToMerge,<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span>      final boolean forcible,<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span>      final long nonceGroup,<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span>      final long nonce) throws IOException {<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span>    checkInitialized();<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span><a name="line.1859"></a>
-<span class="sourceLineNo">1860</span>    assert(regionsToMerge.length == 2);<a name="line.1860"></a>
+<span class="sourceLineNo">1802</span>      Collections.shuffle(allEnabledTables);<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span><a name="line.1803"></a>
+<span class="sourceLineNo">1804</span>      for (TableName table : allEnabledTables) {<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span>        if (isInMaintenanceMode()) {<a name="line.1805"></a>
+<span class="sourceLineNo">1806</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span>          return false;<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span>        }<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span><a name="line.1809"></a>
+<span class="sourceLineNo">1810</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span>          continue;<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span>        }<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span>        if (plans != null) {<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span>          for (NormalizationPlan plan : plans) {<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1821"></a>
+<span class="sourceLineNo">1822</span>              splitPlanCount++;<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span>              mergePlanCount++;<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span>            }<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span>          }<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span>      }<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span>    }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span>    // Return true indicating a success.<a name="line.1831"></a>
+<span class="sourceLineNo">1832</span>    return true;<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span>  }<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span><a name="line.1834"></a>
+<span class="sourceLineNo">1835</span>  /**<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span>   */<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span>  @Override<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span>  public String getClientIdAuditPrefix() {<a name="line.1839"></a>
+<span class="sourceLineNo">1840</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span>  }<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span><a name="line.1843"></a>
+<span class="sourceLineNo">1844</span>  /**<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span>   * Switch for the background CatalogJanitor thread.<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span>   * if disabled.<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span>   */<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1850"></a>
+<span class="sourceLineNo">1851</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span>  }<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span><a name="line.1853"></a>
+<span class="sourceLineNo">1854</span>  @Override<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span>  public long mergeRegions(<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span>      final RegionInfo[] regionsToMerge,<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span>      final boolean forcible,<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span>      final long nonceGroup,<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span>      final long nonce) throws IOException {<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span>    checkInitialized();<a name="line.1860"></a>
 <span class="sourceLineNo">1861</span><a name="line.1861"></a>
-<span class="sourceLineNo">1862</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span>    }<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span><a name="line.1866"></a>
-<span class="sourceLineNo">1867</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span>      throw new IOException (<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1869"></a>
-<span class="sourceLineNo">1870</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span>    }<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span><a name="line.1872"></a>
-<span class="sourceLineNo">1873</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span>      throw new MergeRegionException(<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span>    }<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span><a name="line.1877"></a>
-<span class="sourceLineNo">1878</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1878"></a>
-<span class="sourceLineNo">1879</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span>      @Override<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span>      protected void run() throws IOException {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span><a name="line.1883"></a>
-<span class="sourceLineNo">1884</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span><a name="line.1886"></a>
-<span class="sourceLineNo">1887</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span>          regionsToMerge, forcible));<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span><a name="line.1889"></a>
-<span class="sourceLineNo">1890</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span>      }<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span><a name="line.1892"></a>
-<span class="sourceLineNo">1893</span>      @Override<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span>      protected String getDescription() {<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span>        return "MergeTableProcedure";<a name="line.1895"></a>
-<span class="sourceLineNo">1896</span>      }<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span>    });<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span>  }<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span>  @Override<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1901"></a>
-<span class="sourceLineNo">1902</span>      final long nonceGroup, final long nonce)<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span>  throws IOException {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span>    checkInitialized();<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1905"></a>
-<span class="sourceLineNo">1906</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>      @Override<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>      protected void run() throws IOException {<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span><a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>        // Execute the operation asynchronously<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span><a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>      @Override<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span>      protected String getDescription() {<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>        return "SplitTableProcedure";<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>      }<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>    });<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>  // Replace with an async implementation from which you can get<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span>  // a success/failure result.<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>  @VisibleForTesting<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span><a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    RegionInfo hri;<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>    if (regionState != null) {<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>      hri = regionState.getRegion();<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>    } else {<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span><a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>    ServerName dest;<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>        : new ArrayList&lt;&gt;(1);<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>      LOG.info(<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>              + " because the server is in exclude list");<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>      destServerName = null;<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>    }<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>        "choosing a server at random");<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>      exclude.add(regionState.getServerName());<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>      if (dest == null) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>        return;<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span>      }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    } else {<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span>      if (dest == null) {<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>        return;<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>      }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span>      // TODO: What is this? I don't get it.<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>        // regions on master.<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>          + " because it should not be on master");<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>        return;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span>      }<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    }<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span><a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>      return;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>    }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>    // Now we can do the move<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span><a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>    try {<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      checkInitialized();<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>      if (this.cpHost != null) {<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>      }<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span>      // closed<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span><a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>      try {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        // Is this going to work? Will we throw exception on error?<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        future.get();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>        throw new HBaseIOException(e);<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>      }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span>      if (this.cpHost != null) {<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>    } catch (IOException ioe) {<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      if (ioe instanceof HBaseIOException) {<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        throw (HBaseIOException)ioe;<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      }<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>      throw new HBaseIOException(ioe);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span>  }<a name="line.2014"></a>
-<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>  @Override<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>  public long createTable(<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>      final TableDescriptor tableDescriptor,<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>      final byte [][] splitKeys,<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>      final long nonceGroup,<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span>      final long nonce) throws IOException {<a name="line.2021"></a>
-<span class="sourceLineNo">2022</span>    checkInitialized();<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span><a name="line.2023"></a>
-<span class="sourceLineNo">2024</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span><a name="line.2026"></a>
-<span class="sourceLineNo">2027</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2027"></a>
-<span class="sourceLineNo">2028</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2028"></a>
-<span class="sourceLineNo">2029</span><a name="line.2029"></a>
-<span class="sourceLineNo">2030</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span>      @Override<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span>      protected void run() throws IOException {<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2034"></a>
-<span class="sourceLineNo">2035</span><a name="line.2035"></a>
-<span class="sourceLineNo">2036</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2036"></a>
+<span class="sourceLineNo">1862</span>    assert(regionsToMerge.length == 2);<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span><a name="line.1863"></a>
+<span class="sourceLineNo">1864</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span><a name="line.1868"></a>
+<span class="sourceLineNo">1869</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span>      throw new IOException (<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span>    }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span><a name="line.1874"></a>
+<span class="sourceLineNo">1875</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span>      throw new MergeRegionException(<a name="line.1876"></a>
+<span class="sourceLineNo">1877</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span>    }<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span><a name="line.1879"></a>
+<span class="sourceLineNo">1880</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span>      @Override<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span>      protected void run() throws IOException {<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span><a name="line.1888"></a>
+<span class="sourceLineNo">1889</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span>          regionsToMerge, forcible));<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span><a name="line.1891"></a>
+<span class="sourceLineNo">1892</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span>      }<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span><a name="line.1894"></a>
+<span class="sourceLineNo">1895</span>      @Override<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span>      protected String getDescription() {<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span>        return "MergeTableProcedure";<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span>      }<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span>    });<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span>  }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span><a name="line.1901"></a>
+<span class="sourceLineNo">1902</span>  @Override<a name="line.1902"></a>
+<span class="sourceLineNo">1903</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span>      final long nonceGroup, final long nonce)<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span>  throws IOException {<a name="line.1905"></a>
+<span class="sourceLineNo">1906</span>    checkInitialized();<a name="line.1906"></a>
+<span class="sourceLineNo">1907</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>      @Override<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>      protected void run() throws IOException {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        // Execute the operation asynchronously<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>      }<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>      @Override<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>      protected String getDescription() {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span>        return "SplitTableProcedure";<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>      }<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>    });<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  }<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span><a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span>  // Replace with an async implementation from which you can get<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>  // a success/failure result.<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span>  @VisibleForTesting<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    RegionInfo hri;<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    if (regionState != null) {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      hri = regionState.getRegion();<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    } else {<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span>    }<a name="line.1938"></a>
+<span class="sourceLineNo">1939</span><a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>    ServerName dest;<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>        : new ArrayList&lt;&gt;(1);<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>      LOG.info(<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span>              + " because the server is in exclude list");<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>      destServerName = null;<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    }<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span>        "choosing a server at random");<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>      exclude.add(regionState.getServerName());<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span>      if (dest == null) {<a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        return;<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>      }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span>    } else {<a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>      if (dest == null) {<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>        return;<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      }<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>      // TODO: What is this? I don't get it.<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>        // regions on master.<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>          + " because it should not be on master");<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>        return;<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>      }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span>    }<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span><a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>      return;<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>    }<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span><a name="line.1983"></a>
+<span class="sourceLineNo">1984</span>    // Now we can do the move<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span><a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>    try {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      checkInitialized();<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      if (this.cpHost != null) {<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span>      }<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      // closed<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span><a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      try {<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>        // Is this going to work? Will we throw exception on error?<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        future.get();<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        throw new HBaseIOException(e);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>      if (this.cpHost != null) {<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>      }<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>    } catch (IOException ioe) {<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>      if (ioe instanceof HBaseIOException) {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>        throw (HBaseIOException)ioe;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>      }<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      throw new HBaseIOException(ioe);<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>    }<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>  }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>  @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>  public long createTable(<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      final TableDescriptor tableDescriptor,<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>      final byte [][] splitKeys,<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>      final long nonceGroup,<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>      final long nonce) throws IOException {<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span>    checkInitialized();<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span><a name="line.2025"></a>
+<span class="sourceLineNo">2026</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2027"></a>
+<span class="sourceLineNo">2028</span><a name="line.2028"></a>
+<span class="sourceLineNo">2029</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span><a name="line.2031"></a>
+<span class="sourceLineNo">2032</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span>      @Override<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span>      protected void run() throws IOException {<a name="line.2035"></a>
+<span class="sourceLineNo">2036</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2036"></a>
 <span class="sourceLineNo">2037</span><a name="line.2037"></a>
-<span class="sourceLineNo">2038</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span>        //<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2042"></a>
-<span class="sourceLineNo">2043</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span>        submitProcedure(new CreateTableProcedure(<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span>        latch.await();<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span>      }<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span><a name="line.2050"></a>
-<span class="sourceLineNo">2051</span>      @Override<a name="line.2051"></a>
-<span class="sourceLineNo">2052</span>      protected String getDescription() {<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span>        return "CreateTableProcedure";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span>      }<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span>    });<a name="line.2055"></a>
-<span class="sourceLineNo">2056</span>  }<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span><a name="line.2057"></a>
-<span class="sourceLineNo">2058</span>  @Override<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2059"></a>
-<span class="sourceLineNo">2060</span>    if (isStopped()) {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span>      throw new MasterNotRunningException();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span>    }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span>    if (!(tableName.isSystemTable())) {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span>      throw new IllegalArgumentException(<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span>        "Only system table creation can use this createSystemTable API");<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span>    }<a name="line.2068"></a>
-<span class="sourceLineNo">2069</span><a name="line.2069"></a>
-<span class="sourceLineNo">2070</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2070"></a>
+<span class="sourceLineNo">2038</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span><a name="line.2039"></a>
+<span class="sourceLineNo">2040</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2041"></a>
+<span class="sourceLineNo">2042</span>        //<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span>        submitProcedure(new CreateTableProcedure(<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span>        latch.await();<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span>      }<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span><a name="line.2052"></a>
+<span class="sourceLineNo">2053</span>      @Override<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span>      protected String getDescription() {<a name="line.2054"></a>
+<span class="sourceLineNo">2055</span>        return "CreateTableProcedure";<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span>      }<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span>    });<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span>  }<a name="line.2058"></a>
+<span class="sourceLineNo">2059</span><a name="line.2059"></a>
+<span class="sourceLineNo">2060</span>  @Override<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span>    if (isStopped()) {<a name="line.2062"></a>
+<span class="sourceLineNo">2063</span>      throw new MasterNotRunningException();<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span>    }<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span><a name="line.2065"></a>
+<span class="sourceLineNo">2066</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2066"></a>
+<span class="sourceLineNo">2067</span>    if (!(tableName.isSystemTable())) {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span>      throw new IllegalArgumentException(<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span>        "Only system table creation can use this createSystemTable API");<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span>    }<a name="line.2070"></a>
 <span class="sourceLineNo">2071</span><a name="line.2071"></a>
-<span class="sourceLineNo">2072</span>    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2072"></a>
+<span class="sourceLineNo">2072</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2072"></a>
 <span class="sourceLineNo">2073</span><a name="line.2073"></a>
-<span class="sourceLineNo">2074</span>    // This special create table is called locally to master.  Therefore, no RPC means no need<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span>    // to use nonce to detect duplicated RPC call.<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span>    long procId = this.procedureExecutor.submitProcedure(<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span>      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span><a name="line.2078"></a>
-<span class="sourceLineNo">2079</span>    return procId;<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span>  }<a name="line.2080"></a>
-<span class="sourceLineNo">2081</span><a name="line.2081"></a>
-<span class="sourceLineNo">2082</span>  /**<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span>   * Checks whether the table conforms to some sane limits, and configured<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span>   * values (compression, etc) work. Throws an exception if something is wrong.<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span>   * @throws IOException<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span>   */<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span>  private void sanityCheckTableDescriptor(final TableDescriptor htd) throws IOException {<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span>    final String CONF_KEY = "hbase.table.sanity.checks";<a name="line.2088"></a>
-<span class="sourceLineNo">2089</span>    boolean logWarn = false;<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span>    if (!conf.getBoolean(CONF_KEY, true)) {<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span>      logWarn = true;<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span>    }<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span>    String tableVal = htd.getValue(CONF_KEY);<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span>    if (tableVal != null &amp;&amp; !Boolean.valueOf(tableVal)) {<a name="line.2094"></a>
-<span class="sourceLineNo">2095</span>      logWarn = true;<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span>    }<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span><a name="line.2097"></a>
-<span class="sourceLineNo">2098</span>    // check max file size<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span>    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span>    long maxFileSize = htd.getMaxFileSize();<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span>    if (maxFileSize &lt; 0) {<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span>      maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span>    }<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span>    if (maxFileSize &lt; conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span>      String message = "MAX_FILESIZE for table descriptor or "<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span>          + "\"hbase.hregion.max.filesize\" (" + maxFileSize<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span>          + ") is too small, which might cause over splitting into unmanageable "<a name="line.2107"></a>
-<span class="sourceLineNo">2108</span>          + "number of regions.";<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span>    }<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span><a name="line.2111"></a>
-<span class="sourceLineNo">2112</span>    // check flush size<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span>    long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span>    long flushSize = htd.getMemStoreFlushSize();<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span>    if (flushSize &lt; 0) {<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span>      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);<a name="line.2116"></a>
-<span class="sourceLineNo">2117</span>    }<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span>    if (flushSize &lt; conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span>      String message = "MEMSTORE_FLUSHSIZE for table descriptor or "<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span>          + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span>          + " very frequent flushing.";<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2122"></a>
-<span class="sourceLineNo">2123</span>    }<a name="line.2123"></a>
-<span class="sourceLineNo">2124</span><a name="line.2124"></a>
-<span class="sourceLineNo">2125</span>    // check that coprocessors and other specified plugin classes can be loaded<a name="line.2125"></a>
-<span class="sourceLineNo">2126</span>    try {<a name="line.2126"></a>
-<span class="sourceLineNo">2127</span>      checkClassLoading(conf, htd);<a name="line.2127"></a>
-<span class="sourceLineNo">2128</span>    } catch (Exception ex) {<a name="line.2128"></a>
-<span class="sourceLineNo">2129</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);<a name="line.2129"></a>
-<span class="sourceLineNo">2130</span>    }<a name="line.2130"></a>
-<span class="sourceLineNo">2131</span><a name="line.2131"></a>
-<span class="sourceLineNo">2132</span>    // check compression can be loaded<a name="line.2132"></a>
-<span class="sourceLineNo">2133</span>    try {<a name="line.2133"></a>
-<span class="sourceLineNo">2134</span>      checkCompression(htd);<a name="line.2134"></a>
-<span class="sourceLineNo">2135</span>    } catch (IOException e) {<a name="line.2135"></a>
-<span class="sourceLineNo">2136</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2136"></a>
-<span class="sourceLineNo">2137</span>    }<a name="line.2137"></a>
-<span class="sourceLineNo">2138</span><a name="line.2138"></a>
-<span class="sourceLineNo">2139</span>    // check encryption can be loaded<a name="line.2139"></a>
-<span class="sourceLineNo">2140</span>    try {<a name="line.2140"></a>
-<span class="sourceLineNo">2141</span>      checkEncryption(conf, htd);<a name="line.2141"></a>
-<span class="sourceLineNo">2142</span>    } catch (IOException e) {<a name="line.2142"></a>
-<span class="sourceLineNo">2143</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2143"></a>
-<span class="sourceLineNo">2144</span>    }<a name="line.2144"></a>
-<span class="sourceLineNo">2145</span>    // Verify compaction policy<a name="line.2145"></a>
-<span class="sourceLineNo">2146</span>    try{<a name="line.2146"></a>
-<span class="sourceLineNo">2147</span>      checkCompactionPolicy(conf, htd);<a name="line.2147"></a>
-<span class="sourceLineNo">2148</span>    } catch(IOException e){<a name="line.2148"></a>
-<span class="sourceLineNo">2149</span>      warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);<a name="line.2149"></a>
-<span class="sourceLineNo">2150</span>    }<a name="line.2150"></a>
-<span class="sourceLineNo">2151</span>    // check that we have at least 1 CF<a name="line.2151"></a>
-<span class="sourceLineNo">2152</span>    if (htd.getColumnFamilyCount() == 0) {<a name="line.2152"></a>
-<span class="sourceLineNo">2153</span>      String message = "Table should have at least one column family.";<a name="line.2153"></a>
-<span class="sourceLineNo">2154</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2154"></a>
-<span class="sourceLineNo">2155</span>    }<a name="line.2155"></a>
-<span class="sourceLineNo">2156</span><a name="line.2156"></a>
-<span class="sourceLineNo">2157</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2157"></a>
-<span class="sourceLineNo">2158</span>      if (hcd.getTimeToLive() &lt;= 0) {<a name="line.2158"></a>
-<span class="sourceLineNo">2159</span>        String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";<a name="line.2159"></a>
-<span class="sourceLineNo">2160</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2160"></a>
-<span class="sourceLineNo">2161</span>      }<a name="line.2161"></a>
-<span class="sourceLineNo">2162</span><a name="line.2162"></a>
-<span class="sourceLineNo">2163</span>      // check blockSize<a name="line.2163"></a>
-<span class="sourceLineNo">2164</span>      if (hcd.getBlocksize() &lt; 1024 || hcd.getBlocksize() &gt; 16 * 1024 * 1024) {<a name="line.2164"></a>
-<span class="sourceLineNo">2165</span>        String message = "Block size for column family " + hcd.getNameAsString()<a name="line.2165"></a>
-<span class="sourceLineNo">2166</span>            + "  must be between 1K and 16MB.";<a name="line.2166"></a>
-<span class="sourceLineNo">2167</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2167"></a>
-<span class="sourceLineNo">2168</span>      }<a name="line.2168"></a>
-<span class="sourceLineNo">2169</span><a name="line.2169"></a>
-<span class="sourceLineNo">2170</span>      // check versions<a name="line.2170"></a>
-<span class="sourceLineNo">2171</span>      if (hcd.getMinVersions() &lt; 0) {<a name="line.2171"></a>
-<span class="sourceLineNo">2172</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2172"></a>
-<span class="sourceLineNo">2173</span>          + "  must be positive.";<a name="line.2173"></a>
-<span class="sourceLineNo">2174</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2174"></a>
-<span class="sourceLineNo">2175</span>      }<a name="line.2175"></a>
-<span class="sourceLineNo">2176</span>      // max versions already being checked<a name="line.2176"></a>
-<span class="sourceLineNo">2177</span><a name="line.2177"></a>
-<span class="sourceLineNo">2178</span>      // HBASE-13776 Setting illegal versions for ColumnFamilyDescriptor<a name="line.2178"></a>
-<span class="sourceLineNo">2179</span>      //  does not throw IllegalArgumentException<a name="line.2179"></a>
-<span class="sourceLineNo">2180</span>      // check minVersions &lt;= maxVerions<a name="line.2180"></a>
-<span class="sourceLineNo">2181</span>      if (hcd.getMinVersions() &gt; hcd.getMaxVersions()) {<a name="line.2181"></a>
-<span class="sourceLineNo">2182</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2182"></a>
-<span class="sourceLineNo">2183</span>            + " must be less than the Max versions.";<a name="line.2183"></a>
-<span class="sourceLineNo">2184</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2184"></a>
-<span class="sourceLineNo">2185</span>      }<a name="line.2185"></a>
-<span class="sourceLineNo">2186</span><a name="line.2186"></a>
-<span class="sourceLineNo">2187</span>      // check replication scope<a name="line.2187"></a>
-<span class="sourceLineNo">2188</span>      checkReplicationScope(hcd);<a name="line.2188"></a>
-<span class="sourceLineNo">2189</span>      // check bloom filter type<a name="line.2189"></a>
-<span class="sourceLineNo">2190</span>      checkBloomFilterType(hcd);<a name="line.2190"></a>
-<span class="sourceLineNo">2191</span><a name="line.2191"></a>
-<span class="sourceLineNo">2192</span>      // check data replication factor, it can be 0(default value) when user has not explicitly<a name="line.2192"></a>
-<span class="sourceLineNo">2193</span>      // set the value, in this case we use default replication factor set in the file system.<a name="line.2193"></a>
-<span class="sourceLineNo">2194</span>      if (hcd.getDFSReplication() &lt; 0) {<a name="line.2194"></a>
-<span class="sourceLineNo">2195</span>        String message = "HFile Replication for column family " + hcd.getNameAsString()<a name="line.2195"></a>
-<span class="sourceLineNo">2196</span>            + "  must be greater than zero.";<a name="line.2196"></a>
-<span class="sourceLineNo">2197</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2197"></a>
-<span class="sourceLineNo">2198</span>      }<a name="line.2198"></a>
-<span class="sourceLineNo">2199</span><a name="line.2199"></a>
-<span class="sourceLineNo">2200</span>      // TODO: should we check coprocessors and encryption ?<a name="line.2200"></a>
-<span class="sourceLineNo">2201</span>    }<a name="line.2201"></a>
-<span class="sourceLineNo">2202</span>  }<a name="line.2202"></a>
-<span class="sourceLineNo">2203</span><a name="line.2203"></a>
-<span class="sourceLineNo">2204</span>  private void checkReplicationScope(ColumnFamilyDescriptor hcd) throws IOException{<a name="line.2204"></a>
-<span class="sourceLineNo">2205</span>    // check replication scope<a name="line.2205"></a>
-<span class="sourceLineNo">2206</span>    WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(hcd.getScope());<a name="line.2206"></a>
-<span class="sourceLineNo">2207</span>    if (scop == null) {<a name="line.2207"></a>
-<span class="sourceLineNo">2208</span>      String message = "Replication scope for column family "<a name="line.2208"></a>
-<span class="sourceLineNo">2209</span>          + hcd.getNameAsString() + " is " + hcd.getScope() + " which is invalid.";<a name="line.2209"></a>
-<span class="sourceLineNo">2210</span><a name="line.2210"></a>
-<span class="sourceLineNo">2211</span>      LOG.error(message);<a name="line.2211"></a>
-<span class="sourceLineNo">2212</span>      throw new DoNotRetryIOException(message);<a name="line.2212"></a>
-<span class="sourceLineNo">2213</span>    }<a name="line.2213"></a>
-<span class="sourceLineNo">2214</span>  }<a name="line.2214"></a>
-<span class="sourceLineNo">2215</span><a name="line.2215"></a>
-<span class="sourceLineNo">2216</span>  private void checkCompactionPolicy(Configuration conf, TableDescriptor htd)<a name="line.2216"></a>
-<span class="sourceLineNo">2217</span>      throws IOException {<a name="line.2217"></a>
-<span class="sourceLineNo">2218</span>    // FIFO compaction has some requirements<a name="line.2218"></a>
-<span class="sourceLineNo">2219</span>    // Actually FCP ignores periodic major compactions<a name="line.2219"></a>
-<span class="sourceLineNo">2220</span>    String className = htd.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2220"></a>
-<span class="sourceLineNo">2221</span>    if (className == null) {<a name="line.2221"></a>
-<span class="sourceLineNo">2222</span>      className =<a name="line.2222"></a>
-<span class="sourceLineNo">2223</span>          conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,<a name="line.2223"></a>
-<span class="sourceLineNo">2224</span>            ExploringCompactionPolicy.class.getName());<a name="line.2224"></a>
-<span class="sourceLineNo">2225</span>    }<a name="line.2225"></a>
-<span class="sourceLineNo">2226</span><a name="line.2226"></a>
-<span class="sourceLineNo">2227</span>    int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;<a name="line.2227"></a>
-<span class="sourceLineNo">2228</span>    String sv = htd.getValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2228"></a>
-<span class="sourceLineNo">2229</span>    if (sv != null) {<a name="line.2229"></a>
-<span class="sourceLineNo">2230</span>      blockingFileCount = Integer.parseInt(sv);<a name="line.2230"></a>
-<span class="sourceLineNo">2231</span>    } else {<a name="line.2231"></a>
-<span class="sourceLineNo">2232</span>      blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);<a name="line.2232"></a>
-<span class="sourceLineNo">2233</span>    }<a name="line.2233"></a>
-<span class="sourceLineNo">2234</span><a name="line.2234"></a>
-<span class="sourceLineNo">2235</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2235"></a>
-<span class="sourceLineNo">2236</span>      String compactionPolicy =<a name="line.2236"></a>
-<span class="sourceLineNo">2237</span>          hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2237"></a>
-<span class="sourceLineNo">2238</span>      if (compactionPolicy == null) {<a name="line.2238"></a>
-<span class="sourceLineNo">2239</span>        compactionPolicy = className;<a name="line.2239"></a>
-<span class="sourceLineNo">2240</span>      }<a name="line.2240"></a>
-<span class="sourceLineNo">2241</span>      if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {<a name="line.2241"></a>
-<span class="sourceLineNo">2242</span>        continue;<a name="line.2242"></a>
-<span class="sourceLineNo">2243</span>      }<a name="line.2243"></a>
-<span class="sourceLineNo">2244</span>      // FIFOCompaction<a name="line.2244"></a>
-<span class="sourceLineNo">2245</span>      String message = null;<a name="line.2245"></a>
-<span class="sourceLineNo">2246</span><a name="line.2246"></a>
-<span class="sourceLineNo">2247</span>      // 1. Check TTL<a name="line.2247"></a>
-<span class="sourceLineNo">2248</span>      if (hcd.getTimeToLive() == ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {<a name="line.2248"></a>
-<span class="sourceLineNo">2249</span>        message = "Default TTL is not supported for FIFO compaction";<a name="line.2249"></a>
-<span class="sourceLineNo">2250</span>        throw new IOException(message);<a name="line.2250"></a>
-<span class="sourceLineNo">2251</span>      }<a name="line.2251"></a>
-<span class="sourceLineNo">2252</span><a name="line.2252"></a>
-<span class="sourceLineNo">2253</span>      // 2. Check min versions<a name="line.2253"></a>
-<span class="sourceLineNo">2254</span>      if (hcd.getMinVersions() &gt; 0) {<a name="line.2254"></a>
-<span class="sourceLineNo">2255</span>        message = "MIN_VERSION &gt; 0 is not supported for FIFO compaction";<a name="line.2255"></a>
-<span class="sourceLineNo">2256</span>        throw new IOException(message);<a name="line.2256"></a>
-<span class="sourceLineNo">2257</span>      }<a name="line.2257"></a>
-<span class="sourceLineNo">2258</span><a name="line.2258"></a>
-<span class="sourceLineNo">2259</span>      // 3. blocking file count<a name="line.2259"></a>
-<span class="sourceLineNo">2260</span>      sv = hcd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2260"></a>
-<span class="sourceLineNo">2261</span>      if (sv != null) {<a name="line.2261"></a>
-<span class="sourceLineNo">2262</span>        blockingFileCount = Integer.parseInt(sv);<a name="line.2262"></a>
-<span class="sourceLineNo">2263</span>      }<a name="line.2263"></a>
-<span class="sourceLineNo">2264</span>      if (blockingFileCount &lt; 1000) {<a name="line.2264"></a>
-<span class="sourceLineNo">2265</span>        message =<a name="line.2265"></a>
-<span class="sourceLineNo">2266</span>            "Blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount<a name="line.2266"></a>
-<span class="sourceLineNo">2267</span>                + " is below recommended minimum of 1000 for column family "+ hcd.getNameAsString();<a name="line.2267"></a>
-<span class="sourceLineNo">2268</span>        throw new IOException(message);<a name="line.2268"></a>
-<span class="sourceLineNo">2269</span>      }<a name="line.2269"></a>
-<span class="sourceLineNo">2270</span>    }<a name="line.2270"></a>
-<span class="sourceLineNo">2271</span>  }<a name="line.2271"></a>
-<span class="sourceLineNo">2272</span><a name="line.2272"></a>
-<span class="sourceLineNo">2273</span>  private static void checkBloomFilterType(ColumnFamilyDescriptor cfd)<a name="line.2273"></a>
-<span class="sourceLineNo">2274</span>      throws IOException {<a name="line.2274"></a>
-<span class="sourceLineNo">2275</span>    Configuration conf = new CompoundConfiguration().addStringMap(cfd.getConfiguration());<a name="line.2275"></a>
-<span class="sourceLineNo">2276</span>    try {<a name="line.2276"></a>
-<span class="sourceLineNo">2277</span>      BloomFilterUtil.getBloomFilterParam(cfd.getBloomFilterType(), conf);<a name="line.2277"></a>
-<span class="sourceLineNo">2278</span>    } catch (IllegalArgumentException e) {<a name="line.2278"></a>
-<span class="sourceLineNo">2279</span>      throw new DoNotRetryIOException("Failed to get bloom filter param", e);<a name="line.2279"></a>
-<span class="sourceLineNo">2280</span>    }<a name="line.2280"></a>
-<span class="sourceLineNo">2281</span>  }<a name="line.2281"></a>
-<span class="sourceLineNo">2282</span><a name="line.2282"></a>
-<span class="sourceLineNo">2283</span>  // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.<a name="line.2283"></a>
-<span class="sourceLineNo">2284</span>  private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,<a name="line.2284"></a>
-<span class="sourceLineNo">2285</span>      String message, Exception cause) throws IOException {<a name="line.2285"></a>
-<span class="sourceLineNo">2286</span>    if (!logWarn) {<a name="line.2286"></a>
-<span class="sourceLineNo">2287</span>      throw new DoNotRetryIOException(message + " Set " + confKey +<a name="line.2287"></a>
-<span class="sourceLineNo">2288</span>          " to false at conf or table descriptor if you want to bypass sanity checks", cause);<a name="line.2288"></a>
-<span class="sourceLineNo">2289</span>    }<a name="line.2289"></a>
-<span class="sourceLineNo">2290</span>    LOG.warn(message);<a name="line.2290"></a>
-<span class="sourceLineNo">2291</span>  }<a name="line.2291"></a>
-<span class="sourceLineNo">2292</span><a name="line.2292"></a>
-<span class="sourceLineNo">2293</span>  private void startActiveMasterManager(int infoPort) throws KeeperException {<a name="line.2293"></a>
-<span class="sourceLineNo">2294</span>    String backupZNode = ZNodePaths.joinZNode(<a name="line.2294"></a>
-<span class="sourceLineNo">2295</span>      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());<a name="line.2295"></a>
-<span class="sourceLineNo">2296</span>    /*<a name="line.2296"></a>
-<span class="sourceLineNo">2297</span>    * Add a ZNode for ourselves in the backup master directory since we<a name="line.2297"></a>
-<span class="sourceLineNo">2298</span>    * may not become the active master. If so, we want the actual active<a name="line.2298"></a>
-<span class="sourceLineNo">2299</span>    * master to know we are backup masters, so that it won't assign<a name="line.2299"></a>
-<span class="sourceLineNo">2300</span>    * regions to us if so configured.<a name="line.2300"></a>
-<span class="sourceLineNo">2301</span>    *<a name="line.2301"></a>
-<span class="sourceLineNo">2302</span>    * If we become the active master later, ActiveMasterManager will delete<a name="line.2302"></a>
-<span class="sourceLineNo">2303</span>    * this node explicitly.  If we crash before then, ZooKeeper will delete<a name="line.2303"></a>
-<span class="sourceLineNo">2304</span>    * this node for us since it is ephemeral.<a name="line.2304"></a>
-<span class="sourceLineNo">2305</span>    */<a name="line.2305"></a>
-<span class="sourceLineNo">2306</span>    LOG.info("Adding backup master ZNode " + backupZNode);<a name="line.2306"></a>
-<span class="sourceLineNo">2307</span>    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {<a name="line.2307"></a>
-<span class="sourceLineNo">2308</span>      LOG.warn("Failed create of " + backupZNode + " by " + serverName);<a name="line.2308"></a>
-<span class="sourceLineNo">2309</span>    }<a name="line.2309"></a>
-<span class="sourceLineNo">2310</span>    this.activeMasterManager.setInfoPort(infoPort);<a name="line.2310"></a>
-<span class="sourceLineNo">2311</span>    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);<a name="line.2311"></a>
-<span class="sourceLineNo">2312</span>    // If we're a backup master, stall until a primary to write this address<a name="line.2312"></a>
-<span class="sourceLineNo">2313</span>    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {<a name="line.2313"></a>
-<span class="sourceLineNo">2314</span>      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");<a name="line.2314"></a>
-<span class="sourceLineNo">2315</span>      // This will only be a minute or so while the cluster starts up,<a name="line.2315"></a>
-<span class="sourceLineNo">2316</span>      // so don't worry about setting watches on the parent znode<a name="line.2316"></a>
-<span class="sourceLineNo">2317</span>      while (!activeMasterManager.hasActiveMaster()) {<a name="line.2317"></a>
-<span class="sourceLineNo">2318</span>        LOG.debug("Waiting for master address and cluster state znode to be written.");<a name="line.2318"></a>
-<span class="sourceLineNo">2319</span>        Threads.sleep(timeout);<a name="line.2319"></a>
-<span class="sourceLineNo">2320</span>      }<a name="line.2320"></a>
-<span class="sourceLineNo">2321</span>    }<a name="line.2321"></a>
-<span class="sourceLineNo">2322</span>    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");<a name="line.2322"></a>
-<span class="sourceLineNo">2323</span>    status.setDescription("Master startup");<a name="line.2323"></a>
-<span class="sourceLineNo">2324</span>    try {<a name="line.2324"></a>
-<span class="sourceLineNo">2325</span>      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {<a name="line.2325"></a>
-<span class="sourceLineNo">2326</span>        finishActiveMasterInitialization(status);<a name="line.2326"></a>
-<span class="sourceLineNo">2327</span>      }<a name="line.2327"></a>
-<span class="sourceLineNo">2328</span>    } catch (Throwable t) {<a name="line.2328"></a>
-<span class="sourceLineNo">2329</span>      status.setStatus("Failed to become active: " + t.getMessage());<a name="line.2329"></a>
-<span class="sourceLineNo">2330</span>      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);<a name="line.2330"></a>
-<span class="sourceLineNo">2331</span>      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility<a name="line.2331"></a>
-<span class="sourceLineNo">2332</span>      if (t instanceof NoClassDefFoundError &amp;&amp; t.getMessage().<a name="line.2332"></a>
-<span class="sourceLineNo">2333</span>          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {<a name="line.2333"></a>
-<span class="sourceLineNo">2334</span>        // improved error message for this special case<a name="line.2334"></a>
-<span class="sourceLineNo">2335</span>        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +<a name="line.2335"></a>
-<span class="sourceLineNo">2336</span>          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +<a name="line.2336"></a>
-<span class="sourceLineNo">2337</span>          " or change your hadoop jars to start properly", t);<a name="line.2337"></a>
-<span class="sourceLineNo">2338</span>      } else {<a name="line.2338"></a>
-<span class="sourceLineNo">2339</span>        abort("Unhandled exception. Starting shutdown.", t);<a name="line.2339"></a>
-<span class="sourceLineNo">2340</span>      }<a name="line.2340"></a>
-<span class="sourceLineNo">2341</span>    } finally {<a name="line.2341"></a>
-<span class="sourceLineNo">2342</span>      status.cleanup();<a name="line.2342"></a>
-<span class="sourceLineNo">2343</span>    }<a name="line.2343"></a>
-<span class="sourceLineNo">2344</span>  }<a name="line.2344"></a>
-<span class="sourceLineNo">2345</span><a name="line.2345"></a>
-<span class="sourceLineNo">2346</span>  private void checkCompression(final TableDescriptor htd)<a name="line.2346"></a>
-<span class="sourceLineNo">2347</span>  throws IOException {<a name="line.2347"></a>
-<span class="sourceLineNo">2348</span>    if (!this.masterCheckCompression) return;<a name="line.2348"></a>
-<span class="sourceLineNo">2349</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2349"></a>
-<span class="sourceLineNo">2350</span>      checkCompression(hcd);<a name="line.2350"></a>
-<span class="sourceLineNo">2351</span>    }<a name="line.2351"></a>
-<span class="sourceLineNo">2352</span>  }<a name="line.2352"></a>
-<span class="sourceLineNo">2353</span><a name="line.2353"></a>
-<span class="sourceLineNo">2354</span>  private void checkCompression(final ColumnFamilyDescriptor hcd)<a name="line.2354"></a>
-<span class="sourceLineNo">2355</span>  throws IOException {<a name="line.2355"></a>
-<span class="sourceLineNo">2356</span>    if (!this.masterCheckCompression) return;<a name="line.2356"></a>
-<span class="sourceLineNo">2357</span>    CompressionTest.testCompression(hcd.getCompressionType());<a name="line.2357"></a>
-<span class="sourceLineNo">2358</span>    CompressionTest.testCompression(hcd.getCompactionCompressionType());<a name="line.23

<TRUNCATED>

[43/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html b/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
index f594c43..8a2815f 100644
--- a/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
@@ -142,9 +142,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/Permission.Action.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">Permission.Action</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/AccessControlFilter.Strategy.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">AccessControlFilter.Strategy</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/AccessController.OpType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">AccessController.OpType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/Permission.Action.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">Permission.Action</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
index 29d7634..67b7e3a 100644
--- a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
@@ -191,9 +191,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslStatus.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslStatus</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslUtil.QualityOfProtection.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslUtil.QualityOfProtection</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/AuthMethod.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">AuthMethod</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslUtil.QualityOfProtection.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslUtil.QualityOfProtection</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslStatus.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslStatus</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
index 2923faf..81a90a9 100644
--- a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
@@ -199,8 +199,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftServerRunner.ImplType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftServerRunner.ImplType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">MetricsThriftServerSourceFactoryImpl.FactoryStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftServerRunner.ImplType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftServerRunner.ImplType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftMetrics.ThriftServerType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftMetrics.ThriftServerType</span></a></li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
index ef4add7..0e11c6c 100644
--- a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
@@ -540,14 +540,14 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/ChecksumType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">ChecksumType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.PureJavaComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.PureJavaComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a>&lt;T&gt;)</li>
 <li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.UnsafeComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.UnsafeComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a>&lt;T&gt;)</li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HBaseFsck.ErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HBaseFsck.ErrorReporter.ERROR_CODE</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PrettyPrinter.Unit.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PrettyPrinter.Unit</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.PureJavaComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.PureJavaComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a>&lt;T&gt;)</li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/IdReadWriteLock.ReferenceType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">IdReadWriteLock.ReferenceType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Order.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Order</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/ChecksumType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">ChecksumType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/IdReadWriteLock.ReferenceType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">IdReadWriteLock.ReferenceType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HBaseFsck.ErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HBaseFsck.ErrorReporter.ERROR_CODE</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
index 7da1822..725499b 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
@@ -18,9 +18,9 @@
 <span class="sourceLineNo">010</span>  public static final String version = "3.0.0-SNAPSHOT";<a name="line.10"></a>
 <span class="sourceLineNo">011</span>  public static final String revision = "";<a name="line.11"></a>
 <span class="sourceLineNo">012</span>  public static final String user = "jenkins";<a name="line.12"></a>
-<span class="sourceLineNo">013</span>  public static final String date = "Tue Oct 23 14:46:14 UTC 2018";<a name="line.13"></a>
+<span class="sourceLineNo">013</span>  public static final String date = "Wed Oct 24 14:45:43 UTC 2018";<a name="line.13"></a>
 <span class="sourceLineNo">014</span>  public static final String url = "git://jenkins-websites1.apache.org/home/jenkins/jenkins-slave/workspace/hbase_generate_website/hbase";<a name="line.14"></a>
-<span class="sourceLineNo">015</span>  public static final String srcChecksum = "45eea89e2b35169f4606f1b3d8b0a2ee";<a name="line.15"></a>
+<span class="sourceLineNo">015</span>  public static final String srcChecksum = "1d6019b1fcb1bac7a8be7ca4d582a50d";<a name="line.15"></a>
 <span class="sourceLineNo">016</span>}<a name="line.16"></a>
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
index 1814633..02f25e8 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.SplitParentFirstComparator.html
@@ -119,355 +119,357 @@
 <span class="sourceLineNo">111</span>  protected void chore() {<a name="line.111"></a>
 <span class="sourceLineNo">112</span>    try {<a name="line.112"></a>
 <span class="sourceLineNo">113</span>      AssignmentManager am = this.services.getAssignmentManager();<a name="line.113"></a>
-<span class="sourceLineNo">114</span>      if (this.enabled.get() &amp;&amp; !this.services.isInMaintenanceMode() &amp;&amp; am != null &amp;&amp;<a name="line.114"></a>
-<span class="sourceLineNo">115</span>        am.isMetaLoaded() &amp;&amp; !am.hasRegionsInTransition()) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span>        scan();<a name="line.116"></a>
-<span class="sourceLineNo">117</span>      } else {<a name="line.117"></a>
-<span class="sourceLineNo">118</span>        LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() +<a name="line.118"></a>
-<span class="sourceLineNo">119</span>          ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am +<a name="line.119"></a>
-<span class="sourceLineNo">120</span>          ", metaLoaded=" + (am != null &amp;&amp; am.isMetaLoaded()) + ", hasRIT=" +<a name="line.120"></a>
-<span class="sourceLineNo">121</span>          (am != null &amp;&amp; am.hasRegionsInTransition()));<a name="line.121"></a>
-<span class="sourceLineNo">122</span>      }<a name="line.122"></a>
-<span class="sourceLineNo">123</span>    } catch (IOException e) {<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      LOG.warn("Failed scan of catalog table", e);<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    }<a name="line.125"></a>
-<span class="sourceLineNo">126</span>  }<a name="line.126"></a>
-<span class="sourceLineNo">127</span><a name="line.127"></a>
-<span class="sourceLineNo">128</span>  /**<a name="line.128"></a>
-<span class="sourceLineNo">129</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.129"></a>
-<span class="sourceLineNo">130</span>   * regions, and an ordered map of split parents.<a name="line.130"></a>
-<span class="sourceLineNo">131</span>   * @return triple of scanned rows, map of merged regions and map of split<a name="line.131"></a>
-<span class="sourceLineNo">132</span>   *         parent regioninfos<a name="line.132"></a>
-<span class="sourceLineNo">133</span>   * @throws IOException<a name="line.133"></a>
-<span class="sourceLineNo">134</span>   */<a name="line.134"></a>
-<span class="sourceLineNo">135</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    getMergedRegionsAndSplitParents() throws IOException {<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    return getMergedRegionsAndSplitParents(null);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>  }<a name="line.138"></a>
-<span class="sourceLineNo">139</span><a name="line.139"></a>
-<span class="sourceLineNo">140</span>  /**<a name="line.140"></a>
-<span class="sourceLineNo">141</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.141"></a>
-<span class="sourceLineNo">142</span>   * regions, and an ordered map of split parents. if the given table name is<a name="line.142"></a>
-<span class="sourceLineNo">143</span>   * null, return merged regions and split parents of all tables, else only the<a name="line.143"></a>
-<span class="sourceLineNo">144</span>   * specified table<a name="line.144"></a>
-<span class="sourceLineNo">145</span>   * @param tableName null represents all tables<a name="line.145"></a>
-<span class="sourceLineNo">146</span>   * @return triple of scanned rows, and map of merged regions, and map of split<a name="line.146"></a>
-<span class="sourceLineNo">147</span>   *         parent regioninfos<a name="line.147"></a>
-<span class="sourceLineNo">148</span>   * @throws IOException<a name="line.148"></a>
-<span class="sourceLineNo">149</span>   */<a name="line.149"></a>
-<span class="sourceLineNo">150</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.150"></a>
-<span class="sourceLineNo">151</span>    getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    final boolean isTableSpecified = (tableName != null);<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    // TODO: Only works with single hbase:meta region currently.  Fix.<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    final AtomicInteger count = new AtomicInteger(0);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    // Keep Map of found split parents.  There are candidates for cleanup.<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    // Use a comparator that has split parents come before its daughters.<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    final Map&lt;RegionInfo, Result&gt; splitParents = new TreeMap&lt;&gt;(new SplitParentFirstComparator());<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    final Map&lt;RegionInfo, Result&gt; mergedRegions = new TreeMap&lt;&gt;(RegionInfo.COMPARATOR);<a name="line.158"></a>
-<span class="sourceLineNo">159</span>    // This visitor collects split parents and counts rows in the hbase:meta table<a name="line.159"></a>
-<span class="sourceLineNo">160</span><a name="line.160"></a>
-<span class="sourceLineNo">161</span>    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {<a name="line.161"></a>
-<span class="sourceLineNo">162</span>      @Override<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      public boolean visit(Result r) throws IOException {<a name="line.163"></a>
-<span class="sourceLineNo">164</span>        if (r == null || r.isEmpty()) return true;<a name="line.164"></a>
-<span class="sourceLineNo">165</span>        count.incrementAndGet();<a name="line.165"></a>
-<span class="sourceLineNo">166</span>        RegionInfo info = MetaTableAccessor.getRegionInfo(r);<a name="line.166"></a>
-<span class="sourceLineNo">167</span>        if (info == null) return true; // Keep scanning<a name="line.167"></a>
-<span class="sourceLineNo">168</span>        if (isTableSpecified<a name="line.168"></a>
-<span class="sourceLineNo">169</span>            &amp;&amp; info.getTable().compareTo(tableName) &gt; 0) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span>          // Another table, stop scanning<a name="line.170"></a>
-<span class="sourceLineNo">171</span>          return false;<a name="line.171"></a>
-<span class="sourceLineNo">172</span>        }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>        if (LOG.isTraceEnabled()) LOG.trace("" + info + " IS-SPLIT_PARENT=" + info.isSplitParent());<a name="line.173"></a>
-<span class="sourceLineNo">174</span>        if (info.isSplitParent()) splitParents.put(info, r);<a name="line.174"></a>
-<span class="sourceLineNo">175</span>        if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span>          mergedRegions.put(info, r);<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        }<a name="line.177"></a>
-<span class="sourceLineNo">178</span>        // Returning true means "keep scanning"<a name="line.178"></a>
-<span class="sourceLineNo">179</span>        return true;<a name="line.179"></a>
-<span class="sourceLineNo">180</span>      }<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    };<a name="line.181"></a>
-<span class="sourceLineNo">182</span><a name="line.182"></a>
-<span class="sourceLineNo">183</span>    // Run full scan of hbase:meta catalog table passing in our custom visitor with<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    // the start row<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span>    return new Triple&lt;&gt;(count.get(), mergedRegions, splitParents);<a name="line.187"></a>
-<span class="sourceLineNo">188</span>  }<a name="line.188"></a>
-<span class="sourceLineNo">189</span><a name="line.189"></a>
-<span class="sourceLineNo">190</span>  /**<a name="line.190"></a>
-<span class="sourceLineNo">191</span>   * If merged region no longer holds reference to the merge regions, archive<a name="line.191"></a>
-<span class="sourceLineNo">192</span>   * merge region on hdfs and perform deleting references in hbase:meta<a name="line.192"></a>
-<span class="sourceLineNo">193</span>   * @param mergedRegion<a name="line.193"></a>
-<span class="sourceLineNo">194</span>   * @return true if we delete references in merged region on hbase:meta and archive<a name="line.194"></a>
-<span class="sourceLineNo">195</span>   *         the files on the file system<a name="line.195"></a>
-<span class="sourceLineNo">196</span>   * @throws IOException<a name="line.196"></a>
-<span class="sourceLineNo">197</span>   */<a name="line.197"></a>
-<span class="sourceLineNo">198</span>  boolean cleanMergeRegion(final RegionInfo mergedRegion,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      final RegionInfo regionA, final RegionInfo regionB) throws IOException {<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.200"></a>
-<span class="sourceLineNo">201</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.201"></a>
-<span class="sourceLineNo">202</span>    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    TableDescriptor htd = getTableDescriptor(mergedRegion.getTable());<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    HRegionFileSystem regionFs = null;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>    try {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.206"></a>
-<span class="sourceLineNo">207</span>          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    } catch (IOException e) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    }<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    if (regionFs == null || !regionFs.hasReferences(htd)) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span>      LOG.debug("Deleting region " + regionA.getShortNameToLog() + " and "<a name="line.212"></a>
-<span class="sourceLineNo">213</span>          + regionB.getShortNameToLog()<a name="line.213"></a>
-<span class="sourceLineNo">214</span>          + " from fs because merged region no longer holds references");<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),<a name="line.216"></a>
-<span class="sourceLineNo">217</span>          mergedRegion, regionA, regionB));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      // Remove from in-memory states<a name="line.218"></a>
-<span class="sourceLineNo">219</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);<a name="line.219"></a>
-<span class="sourceLineNo">220</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      this.services.getServerManager().removeRegion(regionA);<a name="line.221"></a>
-<span class="sourceLineNo">222</span>      this.services.getServerManager().removeRegion(regionB);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>      return true;<a name="line.223"></a>
-<span class="sourceLineNo">224</span>    }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>    return false;<a name="line.225"></a>
-<span class="sourceLineNo">226</span>  }<a name="line.226"></a>
-<span class="sourceLineNo">227</span><a name="line.227"></a>
-<span class="sourceLineNo">228</span>  /**<a name="line.228"></a>
-<span class="sourceLineNo">229</span>   * Run janitorial scan of catalog &lt;code&gt;hbase:meta&lt;/code&gt; table looking for<a name="line.229"></a>
-<span class="sourceLineNo">230</span>   * garbage to collect.<a name="line.230"></a>
-<span class="sourceLineNo">231</span>   * @return number of archiving jobs started.<a name="line.231"></a>
-<span class="sourceLineNo">232</span>   * @throws IOException<a name="line.232"></a>
-<span class="sourceLineNo">233</span>   */<a name="line.233"></a>
-<span class="sourceLineNo">234</span>  int scan() throws IOException {<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    int result = 0;<a name="line.235"></a>
-<span class="sourceLineNo">236</span><a name="line.236"></a>
-<span class="sourceLineNo">237</span>    try {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>      if (!alreadyRunning.compareAndSet(false, true)) {<a name="line.238"></a>
-<span class="sourceLineNo">239</span>        LOG.debug("CatalogJanitor already running");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        return result;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt; scanTriple =<a name="line.242"></a>
-<span class="sourceLineNo">243</span>        getMergedRegionsAndSplitParents();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      /**<a name="line.244"></a>
-<span class="sourceLineNo">245</span>       * clean merge regions first<a name="line.245"></a>
-<span class="sourceLineNo">246</span>       */<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      Map&lt;RegionInfo, Result&gt; mergedRegions = scanTriple.getSecond();<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : mergedRegions.entrySet()) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>        if (this.services.isInMaintenanceMode()) {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>          // Stop cleaning if the master is in maintenance mode<a name="line.250"></a>
-<span class="sourceLineNo">251</span>          break;<a name="line.251"></a>
-<span class="sourceLineNo">252</span>        }<a name="line.252"></a>
-<span class="sourceLineNo">253</span><a name="line.253"></a>
-<span class="sourceLineNo">254</span>        PairOfSameType&lt;RegionInfo&gt; p = MetaTableAccessor.getMergeRegions(e.getValue());<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        RegionInfo regionA = p.getFirst();<a name="line.255"></a>
-<span class="sourceLineNo">256</span>        RegionInfo regionB = p.getSecond();<a name="line.256"></a>
-<span class="sourceLineNo">257</span>        if (regionA == null || regionB == null) {<a name="line.257"></a>
-<span class="sourceLineNo">258</span>          LOG.warn("Unexpected references regionA="<a name="line.258"></a>
-<span class="sourceLineNo">259</span>              + (regionA == null ? "null" : regionA.getShortNameToLog())<a name="line.259"></a>
-<span class="sourceLineNo">260</span>              + ",regionB="<a name="line.260"></a>
-<span class="sourceLineNo">261</span>              + (regionB == null ? "null" : regionB.getShortNameToLog())<a name="line.261"></a>
-<span class="sourceLineNo">262</span>              + " in merged region " + e.getKey().getShortNameToLog());<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        } else {<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          if (cleanMergeRegion(e.getKey(), regionA, regionB)) {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>            result++;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>          }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>        }<a name="line.267"></a>
-<span class="sourceLineNo">268</span>      }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>      /**<a name="line.269"></a>
-<span class="sourceLineNo">270</span>       * clean split parents<a name="line.270"></a>
-<span class="sourceLineNo">271</span>       */<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      Map&lt;RegionInfo, Result&gt; splitParents = scanTriple.getThird();<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span>      // Now work on our list of found parents. See if any we can clean up.<a name="line.274"></a>
-<span class="sourceLineNo">275</span>      // regions whose parents are still around<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      HashSet&lt;String&gt; parentNotCleaned = new HashSet&lt;&gt;();<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : splitParents.entrySet()) {<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (this.services.isInMaintenanceMode()) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          // Stop cleaning if the master is in maintenance mode<a name="line.279"></a>
-<span class="sourceLineNo">280</span>          break;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>        }<a name="line.281"></a>
-<span class="sourceLineNo">282</span><a name="line.282"></a>
-<span class="sourceLineNo">283</span>        if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &amp;&amp;<a name="line.283"></a>
-<span class="sourceLineNo">284</span>            cleanParent(e.getKey(), e.getValue())) {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>          result++;<a name="line.285"></a>
-<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
-<span class="sourceLineNo">287</span>          // We could not clean the parent, so it's daughters should not be<a name="line.287"></a>
-<span class="sourceLineNo">288</span>          // cleaned either (HBASE-6160)<a name="line.288"></a>
-<span class="sourceLineNo">289</span>          PairOfSameType&lt;RegionInfo&gt; daughters =<a name="line.289"></a>
-<span class="sourceLineNo">290</span>              MetaTableAccessor.getDaughterRegions(e.getValue());<a name="line.290"></a>
-<span class="sourceLineNo">291</span>          parentNotCleaned.add(daughters.getFirst().getEncodedName());<a name="line.291"></a>
-<span class="sourceLineNo">292</span>          parentNotCleaned.add(daughters.getSecond().getEncodedName());<a name="line.292"></a>
-<span class="sourceLineNo">293</span>        }<a name="line.293"></a>
-<span class="sourceLineNo">294</span>      }<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      return result;<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    } finally {<a name="line.296"></a>
-<span class="sourceLineNo">297</span>      alreadyRunning.set(false);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>    }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   * Compare HRegionInfos in a way that has split parents sort BEFORE their<a name="line.302"></a>
-<span class="sourceLineNo">303</span>   * daughters.<a name="line.303"></a>
-<span class="sourceLineNo">304</span>   */<a name="line.304"></a>
-<span class="sourceLineNo">305</span>  static class SplitParentFirstComparator implements Comparator&lt;RegionInfo&gt; {<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Comparator&lt;byte[]&gt; rowEndKeyComparator = new Bytes.RowEndKeyComparator();<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    @Override<a name="line.307"></a>
-<span class="sourceLineNo">308</span>    public int compare(RegionInfo left, RegionInfo right) {<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      // This comparator differs from the one RegionInfo in that it sorts<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      // parent before daughters.<a name="line.310"></a>
-<span class="sourceLineNo">311</span>      if (left == null) return -1;<a name="line.311"></a>
-<span class="sourceLineNo">312</span>      if (right == null) return 1;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>      // Same table name.<a name="line.313"></a>
-<span class="sourceLineNo">314</span>      int result = left.getTable().compareTo(right.getTable());<a name="line.314"></a>
-<span class="sourceLineNo">315</span>      if (result != 0) return result;<a name="line.315"></a>
-<span class="sourceLineNo">316</span>      // Compare start keys.<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());<a name="line.317"></a>
-<span class="sourceLineNo">318</span>      if (result != 0) return result;<a name="line.318"></a>
-<span class="sourceLineNo">319</span>      // Compare end keys, but flip the operands so parent comes first<a name="line.319"></a>
-<span class="sourceLineNo">320</span>      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span>      return result;<a name="line.322"></a>
-<span class="sourceLineNo">323</span>    }<a name="line.323"></a>
-<span class="sourceLineNo">324</span>  }<a name="line.324"></a>
-<span class="sourceLineNo">325</span><a name="line.325"></a>
-<span class="sourceLineNo">326</span>  /**<a name="line.326"></a>
-<span class="sourceLineNo">327</span>   * If daughters no longer hold reference to the parents, delete the parent.<a name="line.327"></a>
-<span class="sourceLineNo">328</span>   * @param parent RegionInfo of split offlined parent<a name="line.328"></a>
-<span class="sourceLineNo">329</span>   * @param rowContent Content of &lt;code&gt;parent&lt;/code&gt; row in<a name="line.329"></a>
-<span class="sourceLineNo">330</span>   * &lt;code&gt;metaRegionName&lt;/code&gt;<a name="line.330"></a>
-<span class="sourceLineNo">331</span>   * @return True if we removed &lt;code&gt;parent&lt;/code&gt; from meta table and from<a name="line.331"></a>
-<span class="sourceLineNo">332</span>   * the filesystem.<a name="line.332"></a>
-<span class="sourceLineNo">333</span>   * @throws IOException<a name="line.333"></a>
-<span class="sourceLineNo">334</span>   */<a name="line.334"></a>
-<span class="sourceLineNo">335</span>  boolean cleanParent(final RegionInfo parent, Result rowContent)<a name="line.335"></a>
-<span class="sourceLineNo">336</span>  throws IOException {<a name="line.336"></a>
-<span class="sourceLineNo">337</span>    // Check whether it is a merged region and not clean reference<a name="line.337"></a>
-<span class="sourceLineNo">338</span>    // No necessary to check MERGEB_QUALIFIER because these two qualifiers will<a name="line.338"></a>
-<span class="sourceLineNo">339</span>    // be inserted/deleted together<a name="line.339"></a>
-<span class="sourceLineNo">340</span>    if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>      // wait cleaning merge region first<a name="line.341"></a>
-<span class="sourceLineNo">342</span>      return false;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>    }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>    // Run checks on each daughter split.<a name="line.344"></a>
-<span class="sourceLineNo">345</span>    PairOfSameType&lt;RegionInfo&gt; daughters = MetaTableAccessor.getDaughterRegions(rowContent);<a name="line.345"></a>
-<span class="sourceLineNo">346</span>    Pair&lt;Boolean, Boolean&gt; a = checkDaughterInFs(parent, daughters.getFirst());<a name="line.346"></a>
-<span class="sourceLineNo">347</span>    Pair&lt;Boolean, Boolean&gt; b = checkDaughterInFs(parent, daughters.getSecond());<a name="line.347"></a>
-<span class="sourceLineNo">348</span>    if (hasNoReferences(a) &amp;&amp; hasNoReferences(b)) {<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      String daughterA = daughters.getFirst() != null?<a name="line.349"></a>
-<span class="sourceLineNo">350</span>          daughters.getFirst().getShortNameToLog(): "null";<a name="line.350"></a>
-<span class="sourceLineNo">351</span>      String daughterB = daughters.getSecond() != null?<a name="line.351"></a>
-<span class="sourceLineNo">352</span>          daughters.getSecond().getShortNameToLog(): "null";<a name="line.352"></a>
-<span class="sourceLineNo">353</span>      LOG.debug("Deleting region " + parent.getShortNameToLog() +<a name="line.353"></a>
-<span class="sourceLineNo">354</span>        " because daughters -- " + daughterA + ", " + daughterB +<a name="line.354"></a>
-<span class="sourceLineNo">355</span>        " -- no longer hold references");<a name="line.355"></a>
-<span class="sourceLineNo">356</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.356"></a>
-<span class="sourceLineNo">357</span>      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));<a name="line.357"></a>
-<span class="sourceLineNo">358</span>      // Remove from in-memory states<a name="line.358"></a>
-<span class="sourceLineNo">359</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);<a name="line.359"></a>
-<span class="sourceLineNo">360</span>      this.services.getServerManager().removeRegion(parent);<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      return true;<a name="line.361"></a>
-<span class="sourceLineNo">362</span>    }<a name="line.362"></a>
-<span class="sourceLineNo">363</span>    return false;<a name="line.363"></a>
-<span class="sourceLineNo">364</span>  }<a name="line.364"></a>
-<span class="sourceLineNo">365</span><a name="line.365"></a>
-<span class="sourceLineNo">366</span>  /**<a name="line.366"></a>
-<span class="sourceLineNo">367</span>   * @param p A pair where the first boolean says whether or not the daughter<a name="line.367"></a>
-<span class="sourceLineNo">368</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.368"></a>
-<span class="sourceLineNo">369</span>   * whether the daughter has references to the parent.<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @return True the passed &lt;code&gt;p&lt;/code&gt; signifies no references.<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   */<a name="line.371"></a>
-<span class="sourceLineNo">372</span>  private boolean hasNoReferences(final Pair&lt;Boolean, Boolean&gt; p) {<a name="line.372"></a>
-<span class="sourceLineNo">373</span>    return !p.getFirst() || !p.getSecond();<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  }<a name="line.374"></a>
-<span class="sourceLineNo">375</span><a name="line.375"></a>
-<span class="sourceLineNo">376</span>  /**<a name="line.376"></a>
-<span class="sourceLineNo">377</span>   * Checks if a daughter region -- either splitA or splitB -- still holds<a name="line.377"></a>
-<span class="sourceLineNo">378</span>   * references to parent.<a name="line.378"></a>
-<span class="sourceLineNo">379</span>   * @param parent Parent region<a name="line.379"></a>
-<span class="sourceLineNo">380</span>   * @param daughter Daughter region<a name="line.380"></a>
-<span class="sourceLineNo">381</span>   * @return A pair where the first boolean says whether or not the daughter<a name="line.381"></a>
-<span class="sourceLineNo">382</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.382"></a>
-<span class="sourceLineNo">383</span>   * whether the daughter has references to the parent.<a name="line.383"></a>
-<span class="sourceLineNo">384</span>   * @throws IOException<a name="line.384"></a>
-<span class="sourceLineNo">385</span>   */<a name="line.385"></a>
-<span class="sourceLineNo">386</span>  Pair&lt;Boolean, Boolean&gt; checkDaughterInFs(final RegionInfo parent, final RegionInfo daughter)<a name="line.386"></a>
-<span class="sourceLineNo">387</span>  throws IOException {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>    if (daughter == null)  {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>      return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span><a name="line.391"></a>
-<span class="sourceLineNo">392</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.392"></a>
-<span class="sourceLineNo">393</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.393"></a>
-<span class="sourceLineNo">394</span>    Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());<a name="line.394"></a>
-<span class="sourceLineNo">395</span><a name="line.395"></a>
-<span class="sourceLineNo">396</span>    Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());<a name="line.396"></a>
+<span class="sourceLineNo">114</span>      if (this.enabled.get() &amp;&amp; !this.services.isInMaintenanceMode() &amp;&amp;<a name="line.114"></a>
+<span class="sourceLineNo">115</span>        !this.services.getServerManager().isClusterShutdown() &amp;&amp; am != null &amp;&amp;<a name="line.115"></a>
+<span class="sourceLineNo">116</span>        am.isMetaLoaded() &amp;&amp; !am.hasRegionsInTransition()) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span>        scan();<a name="line.117"></a>
+<span class="sourceLineNo">118</span>      } else {<a name="line.118"></a>
+<span class="sourceLineNo">119</span>        LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() +<a name="line.119"></a>
+<span class="sourceLineNo">120</span>          ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am +<a name="line.120"></a>
+<span class="sourceLineNo">121</span>          ", metaLoaded=" + (am != null &amp;&amp; am.isMetaLoaded()) + ", hasRIT=" +<a name="line.121"></a>
+<span class="sourceLineNo">122</span>          (am != null &amp;&amp; am.hasRegionsInTransition()) + " clusterShutDown=" + this.services<a name="line.122"></a>
+<span class="sourceLineNo">123</span>          .getServerManager().isClusterShutdown());<a name="line.123"></a>
+<span class="sourceLineNo">124</span>      }<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    } catch (IOException e) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span>      LOG.warn("Failed scan of catalog table", e);<a name="line.126"></a>
+<span class="sourceLineNo">127</span>    }<a name="line.127"></a>
+<span class="sourceLineNo">128</span>  }<a name="line.128"></a>
+<span class="sourceLineNo">129</span><a name="line.129"></a>
+<span class="sourceLineNo">130</span>  /**<a name="line.130"></a>
+<span class="sourceLineNo">131</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.131"></a>
+<span class="sourceLineNo">132</span>   * regions, and an ordered map of split parents.<a name="line.132"></a>
+<span class="sourceLineNo">133</span>   * @return triple of scanned rows, map of merged regions and map of split<a name="line.133"></a>
+<span class="sourceLineNo">134</span>   *         parent regioninfos<a name="line.134"></a>
+<span class="sourceLineNo">135</span>   * @throws IOException<a name="line.135"></a>
+<span class="sourceLineNo">136</span>   */<a name="line.136"></a>
+<span class="sourceLineNo">137</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    getMergedRegionsAndSplitParents() throws IOException {<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return getMergedRegionsAndSplitParents(null);<a name="line.139"></a>
+<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
+<span class="sourceLineNo">141</span><a name="line.141"></a>
+<span class="sourceLineNo">142</span>  /**<a name="line.142"></a>
+<span class="sourceLineNo">143</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.143"></a>
+<span class="sourceLineNo">144</span>   * regions, and an ordered map of split parents. if the given table name is<a name="line.144"></a>
+<span class="sourceLineNo">145</span>   * null, return merged regions and split parents of all tables, else only the<a name="line.145"></a>
+<span class="sourceLineNo">146</span>   * specified table<a name="line.146"></a>
+<span class="sourceLineNo">147</span>   * @param tableName null represents all tables<a name="line.147"></a>
+<span class="sourceLineNo">148</span>   * @return triple of scanned rows, and map of merged regions, and map of split<a name="line.148"></a>
+<span class="sourceLineNo">149</span>   *         parent regioninfos<a name="line.149"></a>
+<span class="sourceLineNo">150</span>   * @throws IOException<a name="line.150"></a>
+<span class="sourceLineNo">151</span>   */<a name="line.151"></a>
+<span class="sourceLineNo">152</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    final boolean isTableSpecified = (tableName != null);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    // TODO: Only works with single hbase:meta region currently.  Fix.<a name="line.155"></a>
+<span class="sourceLineNo">156</span>    final AtomicInteger count = new AtomicInteger(0);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    // Keep Map of found split parents.  There are candidates for cleanup.<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    // Use a comparator that has split parents come before its daughters.<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    final Map&lt;RegionInfo, Result&gt; splitParents = new TreeMap&lt;&gt;(new SplitParentFirstComparator());<a name="line.159"></a>
+<span class="sourceLineNo">160</span>    final Map&lt;RegionInfo, Result&gt; mergedRegions = new TreeMap&lt;&gt;(RegionInfo.COMPARATOR);<a name="line.160"></a>
+<span class="sourceLineNo">161</span>    // This visitor collects split parents and counts rows in the hbase:meta table<a name="line.161"></a>
+<span class="sourceLineNo">162</span><a name="line.162"></a>
+<span class="sourceLineNo">163</span>    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {<a name="line.163"></a>
+<span class="sourceLineNo">164</span>      @Override<a name="line.164"></a>
+<span class="sourceLineNo">165</span>      public boolean visit(Result r) throws IOException {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>        if (r == null || r.isEmpty()) return true;<a name="line.166"></a>
+<span class="sourceLineNo">167</span>        count.incrementAndGet();<a name="line.167"></a>
+<span class="sourceLineNo">168</span>        RegionInfo info = MetaTableAccessor.getRegionInfo(r);<a name="line.168"></a>
+<span class="sourceLineNo">169</span>        if (info == null) return true; // Keep scanning<a name="line.169"></a>
+<span class="sourceLineNo">170</span>        if (isTableSpecified<a name="line.170"></a>
+<span class="sourceLineNo">171</span>            &amp;&amp; info.getTable().compareTo(tableName) &gt; 0) {<a name="line.171"></a>
+<span class="sourceLineNo">172</span>          // Another table, stop scanning<a name="line.172"></a>
+<span class="sourceLineNo">173</span>          return false;<a name="line.173"></a>
+<span class="sourceLineNo">174</span>        }<a name="line.174"></a>
+<span class="sourceLineNo">175</span>        if (LOG.isTraceEnabled()) LOG.trace("" + info + " IS-SPLIT_PARENT=" + info.isSplitParent());<a name="line.175"></a>
+<span class="sourceLineNo">176</span>        if (info.isSplitParent()) splitParents.put(info, r);<a name="line.176"></a>
+<span class="sourceLineNo">177</span>        if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span>          mergedRegions.put(info, r);<a name="line.178"></a>
+<span class="sourceLineNo">179</span>        }<a name="line.179"></a>
+<span class="sourceLineNo">180</span>        // Returning true means "keep scanning"<a name="line.180"></a>
+<span class="sourceLineNo">181</span>        return true;<a name="line.181"></a>
+<span class="sourceLineNo">182</span>      }<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    };<a name="line.183"></a>
+<span class="sourceLineNo">184</span><a name="line.184"></a>
+<span class="sourceLineNo">185</span>    // Run full scan of hbase:meta catalog table passing in our custom visitor with<a name="line.185"></a>
+<span class="sourceLineNo">186</span>    // the start row<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);<a name="line.187"></a>
+<span class="sourceLineNo">188</span><a name="line.188"></a>
+<span class="sourceLineNo">189</span>    return new Triple&lt;&gt;(count.get(), mergedRegions, splitParents);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>  }<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>  /**<a name="line.192"></a>
+<span class="sourceLineNo">193</span>   * If merged region no longer holds reference to the merge regions, archive<a name="line.193"></a>
+<span class="sourceLineNo">194</span>   * merge region on hdfs and perform deleting references in hbase:meta<a name="line.194"></a>
+<span class="sourceLineNo">195</span>   * @param mergedRegion<a name="line.195"></a>
+<span class="sourceLineNo">196</span>   * @return true if we delete references in merged region on hbase:meta and archive<a name="line.196"></a>
+<span class="sourceLineNo">197</span>   *         the files on the file system<a name="line.197"></a>
+<span class="sourceLineNo">198</span>   * @throws IOException<a name="line.198"></a>
+<span class="sourceLineNo">199</span>   */<a name="line.199"></a>
+<span class="sourceLineNo">200</span>  boolean cleanMergeRegion(final RegionInfo mergedRegion,<a name="line.200"></a>
+<span class="sourceLineNo">201</span>      final RegionInfo regionA, final RegionInfo regionB) throws IOException {<a name="line.201"></a>
+<span class="sourceLineNo">202</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.203"></a>
+<span class="sourceLineNo">204</span>    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    TableDescriptor htd = getTableDescriptor(mergedRegion.getTable());<a name="line.205"></a>
+<span class="sourceLineNo">206</span>    HRegionFileSystem regionFs = null;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    try {<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.208"></a>
+<span class="sourceLineNo">209</span>          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    } catch (IOException e) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    }<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    if (regionFs == null || !regionFs.hasReferences(htd)) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      LOG.debug("Deleting region " + regionA.getShortNameToLog() + " and "<a name="line.214"></a>
+<span class="sourceLineNo">215</span>          + regionB.getShortNameToLog()<a name="line.215"></a>
+<span class="sourceLineNo">216</span>          + " from fs because merged region no longer holds references");<a name="line.216"></a>
+<span class="sourceLineNo">217</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.217"></a>
+<span class="sourceLineNo">218</span>      pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),<a name="line.218"></a>
+<span class="sourceLineNo">219</span>          mergedRegion, regionA, regionB));<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      // Remove from in-memory states<a name="line.220"></a>
+<span class="sourceLineNo">221</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);<a name="line.222"></a>
+<span class="sourceLineNo">223</span>      this.services.getServerManager().removeRegion(regionA);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>      this.services.getServerManager().removeRegion(regionB);<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      return true;<a name="line.225"></a>
+<span class="sourceLineNo">226</span>    }<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    return false;<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>  /**<a name="line.230"></a>
+<span class="sourceLineNo">231</span>   * Run janitorial scan of catalog &lt;code&gt;hbase:meta&lt;/code&gt; table looking for<a name="line.231"></a>
+<span class="sourceLineNo">232</span>   * garbage to collect.<a name="line.232"></a>
+<span class="sourceLineNo">233</span>   * @return number of archiving jobs started.<a name="line.233"></a>
+<span class="sourceLineNo">234</span>   * @throws IOException<a name="line.234"></a>
+<span class="sourceLineNo">235</span>   */<a name="line.235"></a>
+<span class="sourceLineNo">236</span>  int scan() throws IOException {<a name="line.236"></a>
+<span class="sourceLineNo">237</span>    int result = 0;<a name="line.237"></a>
+<span class="sourceLineNo">238</span><a name="line.238"></a>
+<span class="sourceLineNo">239</span>    try {<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (!alreadyRunning.compareAndSet(false, true)) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        LOG.debug("CatalogJanitor already running");<a name="line.241"></a>
+<span class="sourceLineNo">242</span>        return result;<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      }<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt; scanTriple =<a name="line.244"></a>
+<span class="sourceLineNo">245</span>        getMergedRegionsAndSplitParents();<a name="line.245"></a>
+<span class="sourceLineNo">246</span>      /**<a name="line.246"></a>
+<span class="sourceLineNo">247</span>       * clean merge regions first<a name="line.247"></a>
+<span class="sourceLineNo">248</span>       */<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      Map&lt;RegionInfo, Result&gt; mergedRegions = scanTriple.getSecond();<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : mergedRegions.entrySet()) {<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        if (this.services.isInMaintenanceMode()) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span>          // Stop cleaning if the master is in maintenance mode<a name="line.252"></a>
+<span class="sourceLineNo">253</span>          break;<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        }<a name="line.254"></a>
+<span class="sourceLineNo">255</span><a name="line.255"></a>
+<span class="sourceLineNo">256</span>        PairOfSameType&lt;RegionInfo&gt; p = MetaTableAccessor.getMergeRegions(e.getValue());<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        RegionInfo regionA = p.getFirst();<a name="line.257"></a>
+<span class="sourceLineNo">258</span>        RegionInfo regionB = p.getSecond();<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        if (regionA == null || regionB == null) {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>          LOG.warn("Unexpected references regionA="<a name="line.260"></a>
+<span class="sourceLineNo">261</span>              + (regionA == null ? "null" : regionA.getShortNameToLog())<a name="line.261"></a>
+<span class="sourceLineNo">262</span>              + ",regionB="<a name="line.262"></a>
+<span class="sourceLineNo">263</span>              + (regionB == null ? "null" : regionB.getShortNameToLog())<a name="line.263"></a>
+<span class="sourceLineNo">264</span>              + " in merged region " + e.getKey().getShortNameToLog());<a name="line.264"></a>
+<span class="sourceLineNo">265</span>        } else {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>          if (cleanMergeRegion(e.getKey(), regionA, regionB)) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span>            result++;<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          }<a name="line.268"></a>
+<span class="sourceLineNo">269</span>        }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      }<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      /**<a name="line.271"></a>
+<span class="sourceLineNo">272</span>       * clean split parents<a name="line.272"></a>
+<span class="sourceLineNo">273</span>       */<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      Map&lt;RegionInfo, Result&gt; splitParents = scanTriple.getThird();<a name="line.274"></a>
+<span class="sourceLineNo">275</span><a name="line.275"></a>
+<span class="sourceLineNo">276</span>      // Now work on our list of found parents. See if any we can clean up.<a name="line.276"></a>
+<span class="sourceLineNo">277</span>      // regions whose parents are still around<a name="line.277"></a>
+<span class="sourceLineNo">278</span>      HashSet&lt;String&gt; parentNotCleaned = new HashSet&lt;&gt;();<a name="line.278"></a>
+<span class="sourceLineNo">279</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : splitParents.entrySet()) {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>        if (this.services.isInMaintenanceMode()) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>          // Stop cleaning if the master is in maintenance mode<a name="line.281"></a>
+<span class="sourceLineNo">282</span>          break;<a name="line.282"></a>
+<span class="sourceLineNo">283</span>        }<a name="line.283"></a>
+<span class="sourceLineNo">284</span><a name="line.284"></a>
+<span class="sourceLineNo">285</span>        if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &amp;&amp;<a name="line.285"></a>
+<span class="sourceLineNo">286</span>            cleanParent(e.getKey(), e.getValue())) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          result++;<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        } else {<a name="line.288"></a>
+<span class="sourceLineNo">289</span>          // We could not clean the parent, so it's daughters should not be<a name="line.289"></a>
+<span class="sourceLineNo">290</span>          // cleaned either (HBASE-6160)<a name="line.290"></a>
+<span class="sourceLineNo">291</span>          PairOfSameType&lt;RegionInfo&gt; daughters =<a name="line.291"></a>
+<span class="sourceLineNo">292</span>              MetaTableAccessor.getDaughterRegions(e.getValue());<a name="line.292"></a>
+<span class="sourceLineNo">293</span>          parentNotCleaned.add(daughters.getFirst().getEncodedName());<a name="line.293"></a>
+<span class="sourceLineNo">294</span>          parentNotCleaned.add(daughters.getSecond().getEncodedName());<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        }<a name="line.295"></a>
+<span class="sourceLineNo">296</span>      }<a name="line.296"></a>
+<span class="sourceLineNo">297</span>      return result;<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    } finally {<a name="line.298"></a>
+<span class="sourceLineNo">299</span>      alreadyRunning.set(false);<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    }<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span><a name="line.302"></a>
+<span class="sourceLineNo">303</span>  /**<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * Compare HRegionInfos in a way that has split parents sort BEFORE their<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * daughters.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  static class SplitParentFirstComparator implements Comparator&lt;RegionInfo&gt; {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    Comparator&lt;byte[]&gt; rowEndKeyComparator = new Bytes.RowEndKeyComparator();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    @Override<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    public int compare(RegionInfo left, RegionInfo right) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span>      // This comparator differs from the one RegionInfo in that it sorts<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // parent before daughters.<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      if (left == null) return -1;<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      if (right == null) return 1;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>      // Same table name.<a name="line.315"></a>
+<span class="sourceLineNo">316</span>      int result = left.getTable().compareTo(right.getTable());<a name="line.316"></a>
+<span class="sourceLineNo">317</span>      if (result != 0) return result;<a name="line.317"></a>
+<span class="sourceLineNo">318</span>      // Compare start keys.<a name="line.318"></a>
+<span class="sourceLineNo">319</span>      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());<a name="line.319"></a>
+<span class="sourceLineNo">320</span>      if (result != 0) return result;<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      // Compare end keys, but flip the operands so parent comes first<a name="line.321"></a>
+<span class="sourceLineNo">322</span>      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());<a name="line.322"></a>
+<span class="sourceLineNo">323</span><a name="line.323"></a>
+<span class="sourceLineNo">324</span>      return result;<a name="line.324"></a>
+<span class="sourceLineNo">325</span>    }<a name="line.325"></a>
+<span class="sourceLineNo">326</span>  }<a name="line.326"></a>
+<span class="sourceLineNo">327</span><a name="line.327"></a>
+<span class="sourceLineNo">328</span>  /**<a name="line.328"></a>
+<span class="sourceLineNo">329</span>   * If daughters no longer hold reference to the parents, delete the parent.<a name="line.329"></a>
+<span class="sourceLineNo">330</span>   * @param parent RegionInfo of split offlined parent<a name="line.330"></a>
+<span class="sourceLineNo">331</span>   * @param rowContent Content of &lt;code&gt;parent&lt;/code&gt; row in<a name="line.331"></a>
+<span class="sourceLineNo">332</span>   * &lt;code&gt;metaRegionName&lt;/code&gt;<a name="line.332"></a>
+<span class="sourceLineNo">333</span>   * @return True if we removed &lt;code&gt;parent&lt;/code&gt; from meta table and from<a name="line.333"></a>
+<span class="sourceLineNo">334</span>   * the filesystem.<a name="line.334"></a>
+<span class="sourceLineNo">335</span>   * @throws IOException<a name="line.335"></a>
+<span class="sourceLineNo">336</span>   */<a name="line.336"></a>
+<span class="sourceLineNo">337</span>  boolean cleanParent(final RegionInfo parent, Result rowContent)<a name="line.337"></a>
+<span class="sourceLineNo">338</span>  throws IOException {<a name="line.338"></a>
+<span class="sourceLineNo">339</span>    // Check whether it is a merged region and not clean reference<a name="line.339"></a>
+<span class="sourceLineNo">340</span>    // No necessary to check MERGEB_QUALIFIER because these two qualifiers will<a name="line.340"></a>
+<span class="sourceLineNo">341</span>    // be inserted/deleted together<a name="line.341"></a>
+<span class="sourceLineNo">342</span>    if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.342"></a>
+<span class="sourceLineNo">343</span>      // wait cleaning merge region first<a name="line.343"></a>
+<span class="sourceLineNo">344</span>      return false;<a name="line.344"></a>
+<span class="sourceLineNo">345</span>    }<a name="line.345"></a>
+<span class="sourceLineNo">346</span>    // Run checks on each daughter split.<a name="line.346"></a>
+<span class="sourceLineNo">347</span>    PairOfSameType&lt;RegionInfo&gt; daughters = MetaTableAccessor.getDaughterRegions(rowContent);<a name="line.347"></a>
+<span class="sourceLineNo">348</span>    Pair&lt;Boolean, Boolean&gt; a = checkDaughterInFs(parent, daughters.getFirst());<a name="line.348"></a>
+<span class="sourceLineNo">349</span>    Pair&lt;Boolean, Boolean&gt; b = checkDaughterInFs(parent, daughters.getSecond());<a name="line.349"></a>
+<span class="sourceLineNo">350</span>    if (hasNoReferences(a) &amp;&amp; hasNoReferences(b)) {<a name="line.350"></a>
+<span class="sourceLineNo">351</span>      String daughterA = daughters.getFirst() != null?<a name="line.351"></a>
+<span class="sourceLineNo">352</span>          daughters.getFirst().getShortNameToLog(): "null";<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      String daughterB = daughters.getSecond() != null?<a name="line.353"></a>
+<span class="sourceLineNo">354</span>          daughters.getSecond().getShortNameToLog(): "null";<a name="line.354"></a>
+<span class="sourceLineNo">355</span>      LOG.debug("Deleting region " + parent.getShortNameToLog() +<a name="line.355"></a>
+<span class="sourceLineNo">356</span>        " because daughters -- " + daughterA + ", " + daughterB +<a name="line.356"></a>
+<span class="sourceLineNo">357</span>        " -- no longer hold references");<a name="line.357"></a>
+<span class="sourceLineNo">358</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.358"></a>
+<span class="sourceLineNo">359</span>      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));<a name="line.359"></a>
+<span class="sourceLineNo">360</span>      // Remove from in-memory states<a name="line.360"></a>
+<span class="sourceLineNo">361</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);<a name="line.361"></a>
+<span class="sourceLineNo">362</span>      this.services.getServerManager().removeRegion(parent);<a name="line.362"></a>
+<span class="sourceLineNo">363</span>      return true;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
+<span class="sourceLineNo">365</span>    return false;<a name="line.365"></a>
+<span class="sourceLineNo">366</span>  }<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span>  /**<a name="line.368"></a>
+<span class="sourceLineNo">369</span>   * @param p A pair where the first boolean says whether or not the daughter<a name="line.369"></a>
+<span class="sourceLineNo">370</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.370"></a>
+<span class="sourceLineNo">371</span>   * whether the daughter has references to the parent.<a name="line.371"></a>
+<span class="sourceLineNo">372</span>   * @return True the passed &lt;code&gt;p&lt;/code&gt; signifies no references.<a name="line.372"></a>
+<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
+<span class="sourceLineNo">374</span>  private boolean hasNoReferences(final Pair&lt;Boolean, Boolean&gt; p) {<a name="line.374"></a>
+<span class="sourceLineNo">375</span>    return !p.getFirst() || !p.getSecond();<a name="line.375"></a>
+<span class="sourceLineNo">376</span>  }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span>  /**<a name="line.378"></a>
+<span class="sourceLineNo">379</span>   * Checks if a daughter region -- either splitA or splitB -- still holds<a name="line.379"></a>
+<span class="sourceLineNo">380</span>   * references to parent.<a name="line.380"></a>
+<span class="sourceLineNo">381</span>   * @param parent Parent region<a name="line.381"></a>
+<span class="sourceLineNo">382</span>   * @param daughter Daughter region<a name="line.382"></a>
+<span class="sourceLineNo">383</span>   * @return A pair where the first boolean says whether or not the daughter<a name="line.383"></a>
+<span class="sourceLineNo">384</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.384"></a>
+<span class="sourceLineNo">385</span>   * whether the daughter has references to the parent.<a name="line.385"></a>
+<span class="sourceLineNo">386</span>   * @throws IOException<a name="line.386"></a>
+<span class="sourceLineNo">387</span>   */<a name="line.387"></a>
+<span class="sourceLineNo">388</span>  Pair&lt;Boolean, Boolean&gt; checkDaughterInFs(final RegionInfo parent, final RegionInfo daughter)<a name="line.388"></a>
+<span class="sourceLineNo">389</span>  throws IOException {<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    if (daughter == null)  {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.391"></a>
+<span class="sourceLineNo">392</span>    }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());<a name="line.396"></a>
 <span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span>    HRegionFileSystem regionFs = null;<a name="line.398"></a>
+<span class="sourceLineNo">398</span>    Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());<a name="line.398"></a>
 <span class="sourceLineNo">399</span><a name="line.399"></a>
-<span class="sourceLineNo">400</span>    try {<a name="line.400"></a>
-<span class="sourceLineNo">401</span>      if (!FSUtils.isExists(fs, daughterRegionDir)) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>        return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.402"></a>
-<span class="sourceLineNo">403</span>      }<a name="line.403"></a>
-<span class="sourceLineNo">404</span>    } catch (IOException ioe) {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>      LOG.error("Error trying to determine if daughter region exists, " +<a name="line.405"></a>
-<span class="sourceLineNo">406</span>               "assuming exists and has references", ioe);<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.407"></a>
-<span class="sourceLineNo">408</span>    }<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>    boolean references = false;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>    TableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());<a name="line.411"></a>
-<span class="sourceLineNo">412</span>    try {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          this.services.getConfiguration(), fs, tabledir, daughter, true);<a name="line.414"></a>
-<span class="sourceLineNo">415</span><a name="line.415"></a>
-<span class="sourceLineNo">416</span>      for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        if ((references = regionFs.hasReferences(family.getNameAsString()))) {<a name="line.417"></a>
-<span class="sourceLineNo">418</span>          break;<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        }<a name="line.419"></a>
-<span class="sourceLineNo">420</span>      }<a name="line.420"></a>
-<span class="sourceLineNo">421</span>    } catch (IOException e) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span>      LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()<a name="line.422"></a>
-<span class="sourceLineNo">423</span>          + ", to: " + parent.getEncodedName() + " assuming has references", e);<a name="line.423"></a>
-<span class="sourceLineNo">424</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.424"></a>
-<span class="sourceLineNo">425</span>    }<a name="line.425"></a>
-<span class="sourceLineNo">426</span>    return new Pair&lt;&gt;(Boolean.TRUE, Boolean.valueOf(references));<a name="line.426"></a>
-<span class="sourceLineNo">427</span>  }<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span>  private TableDescriptor getTableDescriptor(final TableName tableName)<a name="line.429"></a>
-<span class="sourceLineNo">430</span>      throws FileNotFoundException, IOException {<a name="line.430"></a>
-<span class="sourceLineNo">431</span>    return this.services.getTableDescriptors().get(tableName);<a name="line.431"></a>
-<span class="sourceLineNo">432</span>  }<a name="line.432"></a>
-<span class="sourceLineNo">433</span><a name="line.433"></a>
-<span class="sourceLineNo">434</span>  /**<a name="line.434"></a>
-<span class="sourceLineNo">435</span>   * Checks if the specified region has merge qualifiers, if so, try to clean<a name="line.435"></a>
-<span class="sourceLineNo">436</span>   * them<a name="line.436"></a>
-<span class="sourceLineNo">437</span>   * @param region<a name="line.437"></a>
-<span class="sourceLineNo">438</span>   * @return true if the specified region doesn't have merge qualifier now<a name="line.438"></a>
-<span class="sourceLineNo">439</span>   * @throws IOException<a name="line.439"></a>
-<span class="sourceLineNo">440</span>   */<a name="line.440"></a>
-<span class="sourceLineNo">441</span>  public boolean cleanMergeQualifier(final RegionInfo region)<a name="line.441"></a>
-<span class="sourceLineNo">442</span>      throws IOException {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // Get merge regions if it is a merged region and already has merge<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // qualifier<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    Pair&lt;RegionInfo, RegionInfo&gt; mergeRegions = MetaTableAccessor<a name="line.445"></a>
-<span class="sourceLineNo">446</span>        .getRegionsFromMergeQualifier(this.services.getConnection(),<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          region.getRegionName());<a name="line.447"></a>
-<span class="sourceLineNo">448</span>    if (mergeRegions == null<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        || (mergeRegions.getFirst() == null &amp;&amp; mergeRegions.getSecond() == null)) {<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      // It doesn't have merge qualifier, no need to clean<a name="line.450"></a>
-<span class="sourceLineNo">451</span>      return true;<a name="line.451"></a>
-<span class="sourceLineNo">452</span>    }<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    // It shouldn't happen, we must insert/delete these two qualifiers together<a name="line.453"></a>
-<span class="sourceLineNo">454</span>    if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>      LOG.error("Merged region " + region.getRegionNameAsString()<a name="line.455"></a>
-<span class="sourceLineNo">456</span>          + " has only one merge qualifier in META.");<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      return false;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>    }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>    return cleanMergeRegion(region, mergeRegions.getFirst(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        mergeRegions.getSecond());<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span>}<a name="line.462"></a>
+<span class="sourceLineNo">400</span>    HRegionFileSystem regionFs = null;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    try {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      if (!FSUtils.isExists(fs, daughterRegionDir)) {<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.404"></a>
+<span class="sourceLineNo">405</span>      }<a name="line.405"></a>
+<span class="sourceLineNo">406</span>    } catch (IOException ioe) {<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      LOG.error("Error trying to determine if daughter region exists, " +<a name="line.407"></a>
+<span class="sourceLineNo">408</span>               "assuming exists and has references", ioe);<a name="line.408"></a>
+<span class="sourceLineNo">409</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>    }<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>    boolean references = false;<a name="line.412"></a>
+<span class="sourceLineNo">413</span>    TableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());<a name="line.413"></a>
+<span class="sourceLineNo">414</span>    try {<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.415"></a>
+<span class="sourceLineNo">416</span>          this.services.getConfiguration(), fs, tabledir, daughter, true);<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>      for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        if ((references = regionFs.hasReferences(family.getNameAsString()))) {<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          break;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>      }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>    } catch (IOException e) {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>      LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          + ", to: " + parent.getEncodedName() + " assuming has references", e);<a name="line.425"></a>
+<span class="sourceLineNo">426</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.426"></a>
+<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>    return new Pair&lt;&gt;(Boolean.TRUE, Boolean.valueOf(references));<a name="line.428"></a>
+<span class="sourceLineNo">429</span>  }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>  private TableDescriptor getTableDescriptor(final TableName tableName)<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      throws FileNotFoundException, IOException {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>    return this.services.getTableDescriptors().get(tableName);<a name="line.433"></a>
+<span class="sourceLineNo">434</span>  }<a name="line.434"></a>
+<span class="sourceLineNo">435</span><a name="line.435"></a>
+<span class="sourceLineNo">436</span>  /**<a name="line.436"></a>
+<span class="sourceLineNo">437</span>   * Checks if the specified region has merge qualifiers, if so, try to clean<a name="line.437"></a>
+<span class="sourceLineNo">438</span>   * them<a name="line.438"></a>
+<span class="sourceLineNo">439</span>   * @param region<a name="line.439"></a>
+<span class="sourceLineNo">440</span>   * @return true if the specified region doesn't have merge qualifier now<a name="line.440"></a>
+<span class="sourceLineNo">441</span>   * @throws IOException<a name="line.441"></a>
+<span class="sourceLineNo">442</span>   */<a name="line.442"></a>
+<span class="sourceLineNo">443</span>  public boolean cleanMergeQualifier(final RegionInfo region)<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      throws IOException {<a name="line.444"></a>
+<span class="sourceLineNo">445</span>    // Get merge regions if it is a merged region and already has merge<a name="line.445"></a>
+<span class="sourceLineNo">446</span>    // qualifier<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    Pair&lt;RegionInfo, RegionInfo&gt; mergeRegions = MetaTableAccessor<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        .getRegionsFromMergeQualifier(this.services.getConnection(),<a name="line.448"></a>
+<span class="sourceLineNo">449</span>          region.getRegionName());<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    if (mergeRegions == null<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        || (mergeRegions.getFirst() == null &amp;&amp; mergeRegions.getSecond() == null)) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      // It doesn't have merge qualifier, no need to clean<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      return true;<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    // It shouldn't happen, we must insert/delete these two qualifiers together<a name="line.455"></a>
+<span class="sourceLineNo">456</span>    if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      LOG.error("Merged region " + region.getRegionNameAsString()<a name="line.457"></a>
+<span class="sourceLineNo">458</span>          + " has only one merge qualifier in META.");<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      return false;<a name="line.459"></a>
+<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    return cleanMergeRegion(region, mergeRegions.getFirst(),<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        mergeRegions.getSecond());<a name="line.462"></a>
+<span class="sourceLineNo">463</span>  }<a name="line.463"></a>
+<span class="sourceLineNo">464</span>}<a name="line.464"></a>
 
 
 


[18/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaRackCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="lin

<TRUNCATED>

[32/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">835</span>            getWeightedLocality(clu

<TRUNCATED>

[23/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ReadRequestCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sour

<TRUNCATED>

[17/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.ServerLocalityCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<s

<TRUNCATED>

[16/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.StoreFileCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">

<TRUNCATED>

[38/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
index b7a2a38..6591d6c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/HMaster.TableDescriptorGetter.html
@@ -1789,2174 +1789,2176 @@
 <span class="sourceLineNo">1781</span>      LOG.debug("Master has not been initialized, don't run region normalizer.");<a name="line.1781"></a>
 <span class="sourceLineNo">1782</span>      return false;<a name="line.1782"></a>
 <span class="sourceLineNo">1783</span>    }<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span><a name="line.1784"></a>
-<span class="sourceLineNo">1785</span>    if (isInMaintenanceMode()) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span>      return false;<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span>    }<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span><a name="line.1789"></a>
-<span class="sourceLineNo">1790</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1791"></a>
-<span class="sourceLineNo">1792</span>      return false;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span>    }<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span><a name="line.1794"></a>
-<span class="sourceLineNo">1795</span>    synchronized (this.normalizer) {<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span>      // Don't run the normalizer concurrently<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1797"></a>
-<span class="sourceLineNo">1798</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1798"></a>
-<span class="sourceLineNo">1799</span><a name="line.1799"></a>
-<span class="sourceLineNo">1800</span>      Collections.shuffle(allEnabledTables);<a name="line.1800"></a>
+<span class="sourceLineNo">1784</span>    if (this.getServerManager().isClusterShutdown()) {<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span>      LOG.info("Cluster is shutting down, don't run region normalizer.");<a name="line.1785"></a>
+<span class="sourceLineNo">1786</span>      return false;<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span>    }<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span>    if (isInMaintenanceMode()) {<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span>      LOG.info("Master is in maintenance mode, don't run region normalizer.");<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span>      return false;<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span>    }<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span>    if (!this.regionNormalizerTracker.isNormalizerOn()) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span>      LOG.debug("Region normalization is disabled, don't run region normalizer.");<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span>      return false;<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span>    }<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span><a name="line.1796"></a>
+<span class="sourceLineNo">1797</span>    synchronized (this.normalizer) {<a name="line.1797"></a>
+<span class="sourceLineNo">1798</span>      // Don't run the normalizer concurrently<a name="line.1798"></a>
+<span class="sourceLineNo">1799</span>      List&lt;TableName&gt; allEnabledTables = new ArrayList&lt;&gt;(<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span>        this.tableStateManager.getTablesInStates(TableState.State.ENABLED));<a name="line.1800"></a>
 <span class="sourceLineNo">1801</span><a name="line.1801"></a>
-<span class="sourceLineNo">1802</span>      for (TableName table : allEnabledTables) {<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span>        if (isInMaintenanceMode()) {<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span>          return false;<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span>        }<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span><a name="line.1807"></a>
-<span class="sourceLineNo">1808</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span>          continue;<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span>        }<a name="line.1814"></a>
-<span class="sourceLineNo">1815</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span>        if (plans != null) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span>          for (NormalizationPlan plan : plans) {<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span>              splitPlanCount++;<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span>              mergePlanCount++;<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span>            }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span>          }<a name="line.1824"></a>
-<span class="sourceLineNo">1825</span>        }<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span>      }<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span>    }<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span>    // Return true indicating a success.<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span>    return true;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span>  }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span>  /**<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span>   */<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span>  @Override<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span>  public String getClientIdAuditPrefix() {<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span>  }<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span><a name="line.1841"></a>
-<span class="sourceLineNo">1842</span>  /**<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span>   * Switch for the background CatalogJanitor thread.<a name="line.1843"></a>
-<span class="sourceLineNo">1844</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span>   * if disabled.<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span>   */<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span>  }<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span><a name="line.1851"></a>
-<span class="sourceLineNo">1852</span>  @Override<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span>  public long mergeRegions(<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span>      final RegionInfo[] regionsToMerge,<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span>      final boolean forcible,<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span>      final long nonceGroup,<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span>      final long nonce) throws IOException {<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span>    checkInitialized();<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span><a name="line.1859"></a>
-<span class="sourceLineNo">1860</span>    assert(regionsToMerge.length == 2);<a name="line.1860"></a>
+<span class="sourceLineNo">1802</span>      Collections.shuffle(allEnabledTables);<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span><a name="line.1803"></a>
+<span class="sourceLineNo">1804</span>      for (TableName table : allEnabledTables) {<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span>        if (isInMaintenanceMode()) {<a name="line.1805"></a>
+<span class="sourceLineNo">1806</span>          LOG.debug("Master is in maintenance mode, stop running region normalizer.");<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span>          return false;<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span>        }<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span><a name="line.1809"></a>
+<span class="sourceLineNo">1810</span>        TableDescriptor tblDesc = getTableDescriptors().get(table);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span>        if (table.isSystemTable() || (tblDesc != null &amp;&amp;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span>            !tblDesc.isNormalizationEnabled())) {<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span>          LOG.trace("Skipping normalization for {}, as it's either system"<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span>              + " table or doesn't have auto normalization turned on", table);<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span>          continue;<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span>        }<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span>        List&lt;NormalizationPlan&gt; plans = this.normalizer.computePlanForTable(table);<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span>        if (plans != null) {<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span>          for (NormalizationPlan plan : plans) {<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span>            plan.execute(clusterConnection.getAdmin());<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span>            if (plan.getType() == PlanType.SPLIT) {<a name="line.1821"></a>
+<span class="sourceLineNo">1822</span>              splitPlanCount++;<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span>            } else if (plan.getType() == PlanType.MERGE) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span>              mergePlanCount++;<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span>            }<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span>          }<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span>      }<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span>    }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span>    // If Region did not generate any plans, it means the cluster is already balanced.<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span>    // Return true indicating a success.<a name="line.1831"></a>
+<span class="sourceLineNo">1832</span>    return true;<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span>  }<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span><a name="line.1834"></a>
+<span class="sourceLineNo">1835</span>  /**<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span>   * @return Client info for use as prefix on an audit log string; who did an action<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span>   */<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span>  @Override<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span>  public String getClientIdAuditPrefix() {<a name="line.1839"></a>
+<span class="sourceLineNo">1840</span>    return "Client=" + RpcServer.getRequestUserName().orElse(null)<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span>        + "/" + RpcServer.getRemoteAddress().orElse(null);<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span>  }<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span><a name="line.1843"></a>
+<span class="sourceLineNo">1844</span>  /**<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span>   * Switch for the background CatalogJanitor thread.<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span>   * Used for testing.  The thread will continue to run.  It will just be a noop<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span>   * if disabled.<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span>   * @param b If false, the catalog janitor won't do anything.<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span>   */<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span>  public void setCatalogJanitorEnabled(final boolean b) {<a name="line.1850"></a>
+<span class="sourceLineNo">1851</span>    this.catalogJanitorChore.setEnabled(b);<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span>  }<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span><a name="line.1853"></a>
+<span class="sourceLineNo">1854</span>  @Override<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span>  public long mergeRegions(<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span>      final RegionInfo[] regionsToMerge,<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span>      final boolean forcible,<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span>      final long nonceGroup,<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span>      final long nonce) throws IOException {<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span>    checkInitialized();<a name="line.1860"></a>
 <span class="sourceLineNo">1861</span><a name="line.1861"></a>
-<span class="sourceLineNo">1862</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span>    }<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span><a name="line.1866"></a>
-<span class="sourceLineNo">1867</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span>      throw new IOException (<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1869"></a>
-<span class="sourceLineNo">1870</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span>    }<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span><a name="line.1872"></a>
-<span class="sourceLineNo">1873</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span>      throw new MergeRegionException(<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span>    }<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span><a name="line.1877"></a>
-<span class="sourceLineNo">1878</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1878"></a>
-<span class="sourceLineNo">1879</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span>      @Override<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span>      protected void run() throws IOException {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span><a name="line.1883"></a>
-<span class="sourceLineNo">1884</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span><a name="line.1886"></a>
-<span class="sourceLineNo">1887</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span>          regionsToMerge, forcible));<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span><a name="line.1889"></a>
-<span class="sourceLineNo">1890</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span>      }<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span><a name="line.1892"></a>
-<span class="sourceLineNo">1893</span>      @Override<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span>      protected String getDescription() {<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span>        return "MergeTableProcedure";<a name="line.1895"></a>
-<span class="sourceLineNo">1896</span>      }<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span>    });<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span>  }<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span>  @Override<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1901"></a>
-<span class="sourceLineNo">1902</span>      final long nonceGroup, final long nonce)<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span>  throws IOException {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span>    checkInitialized();<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1905"></a>
-<span class="sourceLineNo">1906</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span>      @Override<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span>      protected void run() throws IOException {<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span><a name="line.1911"></a>
-<span class="sourceLineNo">1912</span>        // Execute the operation asynchronously<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span>      }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span><a name="line.1915"></a>
-<span class="sourceLineNo">1916</span>      @Override<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span>      protected String getDescription() {<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span>        return "SplitTableProcedure";<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span>      }<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span>    });<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span>  }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span>  // Replace with an async implementation from which you can get<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span>  // a success/failure result.<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span>  @VisibleForTesting<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span><a name="line.1930"></a>
-<span class="sourceLineNo">1931</span>    RegionInfo hri;<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span>    if (regionState != null) {<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span>      hri = regionState.getRegion();<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span>    } else {<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span>    }<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span><a name="line.1937"></a>
-<span class="sourceLineNo">1938</span>    ServerName dest;<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span>        : new ArrayList&lt;&gt;(1);<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span>      LOG.info(<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span>              + " because the server is in exclude list");<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span>      destServerName = null;<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span>    }<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1948"></a>
-<span class="sourceLineNo">1949</span>        "choosing a server at random");<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span>      exclude.add(regionState.getServerName());<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1952"></a>
-<span class="sourceLineNo">1953</span>      if (dest == null) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span>        return;<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span>      }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span>    } else {<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span>      if (dest == null) {<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span>        return;<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span>      }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span>      // TODO: What is this? I don't get it.<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span>        // regions on master.<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span>          + " because it should not be on master");<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span>        return;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span>      }<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span>    }<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span><a name="line.1975"></a>
-<span class="sourceLineNo">1976</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span>      return;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span>    }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span>    // Now we can do the move<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span><a name="line.1985"></a>
-<span class="sourceLineNo">1986</span>    try {<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span>      checkInitialized();<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span>      if (this.cpHost != null) {<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span>      }<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span>      // closed<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span><a name="line.1995"></a>
-<span class="sourceLineNo">1996</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span>      try {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span>        // Is this going to work? Will we throw exception on error?<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span>        future.get();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span>        throw new HBaseIOException(e);<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span>      }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span>      if (this.cpHost != null) {<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span>      }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span>    } catch (IOException ioe) {<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span>      if (ioe instanceof HBaseIOException) {<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span>        throw (HBaseIOException)ioe;<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span>      }<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span>      throw new HBaseIOException(ioe);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span>    }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span>  }<a name="line.2014"></a>
-<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span>  @Override<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span>  public long createTable(<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span>      final TableDescriptor tableDescriptor,<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span>      final byte [][] splitKeys,<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>      final long nonceGroup,<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span>      final long nonce) throws IOException {<a name="line.2021"></a>
-<span class="sourceLineNo">2022</span>    checkInitialized();<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span><a name="line.2023"></a>
-<span class="sourceLineNo">2024</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span><a name="line.2026"></a>
-<span class="sourceLineNo">2027</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2027"></a>
-<span class="sourceLineNo">2028</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2028"></a>
-<span class="sourceLineNo">2029</span><a name="line.2029"></a>
-<span class="sourceLineNo">2030</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span>      @Override<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span>      protected void run() throws IOException {<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2034"></a>
-<span class="sourceLineNo">2035</span><a name="line.2035"></a>
-<span class="sourceLineNo">2036</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2036"></a>
+<span class="sourceLineNo">1862</span>    assert(regionsToMerge.length == 2);<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span><a name="line.1863"></a>
+<span class="sourceLineNo">1864</span>    TableName tableName = regionsToMerge[0].getTable();<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span>    if (tableName == null || regionsToMerge[1].getTable() == null) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span>      throw new UnknownRegionException ("Can't merge regions without table associated");<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span><a name="line.1868"></a>
+<span class="sourceLineNo">1869</span>    if (!tableName.equals(regionsToMerge[1].getTable())) {<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span>      throw new IOException (<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span>        "Cannot merge regions from two different tables " + regionsToMerge[0].getTable()<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span>        + " and " + regionsToMerge[1].getTable());<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span>    }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span><a name="line.1874"></a>
+<span class="sourceLineNo">1875</span>    if (RegionInfo.COMPARATOR.compare(regionsToMerge[0], regionsToMerge[1]) == 0) {<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span>      throw new MergeRegionException(<a name="line.1876"></a>
+<span class="sourceLineNo">1877</span>        "Cannot merge a region to itself " + regionsToMerge[0] + ", " + regionsToMerge[1]);<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span>    }<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span><a name="line.1879"></a>
+<span class="sourceLineNo">1880</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span>      @Override<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span>      protected void run() throws IOException {<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span>        getMaster().getMasterCoprocessorHost().preMergeRegions(regionsToMerge);<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span>        LOG.info(getClientIdAuditPrefix() + " Merge regions " +<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span>          regionsToMerge[0].getEncodedName() + " and " + regionsToMerge[1].getEncodedName());<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span><a name="line.1888"></a>
+<span class="sourceLineNo">1889</span>        submitProcedure(new MergeTableRegionsProcedure(procedureExecutor.getEnvironment(),<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span>          regionsToMerge, forcible));<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span><a name="line.1891"></a>
+<span class="sourceLineNo">1892</span>        getMaster().getMasterCoprocessorHost().postMergeRegions(regionsToMerge);<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span>      }<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span><a name="line.1894"></a>
+<span class="sourceLineNo">1895</span>      @Override<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span>      protected String getDescription() {<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span>        return "MergeTableProcedure";<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span>      }<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span>    });<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span>  }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span><a name="line.1901"></a>
+<span class="sourceLineNo">1902</span>  @Override<a name="line.1902"></a>
+<span class="sourceLineNo">1903</span>  public long splitRegion(final RegionInfo regionInfo, final byte[] splitRow,<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span>      final long nonceGroup, final long nonce)<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span>  throws IOException {<a name="line.1905"></a>
+<span class="sourceLineNo">1906</span>    checkInitialized();<a name="line.1906"></a>
+<span class="sourceLineNo">1907</span>    return MasterProcedureUtil.submitProcedure(<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span>      @Override<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span>      protected void run() throws IOException {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span>        getMaster().getMasterCoprocessorHost().preSplitRegion(regionInfo.getTable(), splitRow);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span>        LOG.info(getClientIdAuditPrefix() + " split " + regionInfo.getRegionNameAsString());<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span>        // Execute the operation asynchronously<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span>        submitProcedure(getAssignmentManager().createSplitProcedure(regionInfo, splitRow));<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span>      }<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span>      @Override<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span>      protected String getDescription() {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span>        return "SplitTableProcedure";<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span>      }<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span>    });<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span>  }<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span><a name="line.1924"></a>
+<span class="sourceLineNo">1925</span>  // Public so can be accessed by tests. Blocks until move is done.<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span>  // Replace with an async implementation from which you can get<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span>  // a success/failure result.<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span>  @VisibleForTesting<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span>  public void move(final byte[] encodedRegionName, byte[] destServerName) throws HBaseIOException {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span>    RegionState regionState = assignmentManager.getRegionStates().<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span>      getRegionState(Bytes.toString(encodedRegionName));<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span>    RegionInfo hri;<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span>    if (regionState != null) {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span>      hri = regionState.getRegion();<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span>    } else {<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span>      throw new UnknownRegionException(Bytes.toStringBinary(encodedRegionName));<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span>    }<a name="line.1938"></a>
+<span class="sourceLineNo">1939</span><a name="line.1939"></a>
+<span class="sourceLineNo">1940</span>    ServerName dest;<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span>    List&lt;ServerName&gt; exclude = hri.getTable().isSystemTable() ? assignmentManager.getExcludedServersForSystemTable()<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span>        : new ArrayList&lt;&gt;(1);<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span>    if (destServerName != null &amp;&amp; exclude.contains(ServerName.valueOf(Bytes.toString(destServerName)))) {<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span>      LOG.info(<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span>          Bytes.toString(encodedRegionName) + " can not move to " + Bytes.toString(destServerName)<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span>              + " because the server is in exclude list");<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span>      destServerName = null;<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span>    }<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span>    if (destServerName == null || destServerName.length == 0) {<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span>      LOG.info("Passed destination servername is null/empty so " +<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span>        "choosing a server at random");<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span>      exclude.add(regionState.getServerName());<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span>      final List&lt;ServerName&gt; destServers = this.serverManager.createDestinationServersList(exclude);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span>      dest = balancer.randomAssignment(hri, destServers);<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span>      if (dest == null) {<a name="line.1955"></a>
+<span class="sourceLineNo">1956</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span>        return;<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span>      }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span>    } else {<a name="line.1959"></a>
+<span class="sourceLineNo">1960</span>      ServerName candidate = ServerName.valueOf(Bytes.toString(destServerName));<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span>      dest = balancer.randomAssignment(hri, Lists.newArrayList(candidate));<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span>      if (dest == null) {<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span>        LOG.debug("Unable to determine a plan to assign " + hri);<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span>        return;<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span>      }<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span>      // TODO: What is this? I don't get it.<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span>      if (dest.equals(serverName) &amp;&amp; balancer instanceof BaseLoadBalancer<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span>          &amp;&amp; !((BaseLoadBalancer)balancer).shouldBeOnMaster(hri)) {<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span>        // To avoid unnecessary region moving later by balancer. Don't put user<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span>        // regions on master.<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span>        LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span>          + " to avoid unnecessary region moving later by load balancer,"<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span>          + " because it should not be on master");<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span>        return;<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span>      }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span>    }<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span><a name="line.1977"></a>
+<span class="sourceLineNo">1978</span>    if (dest.equals(regionState.getServerName())) {<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span>      LOG.debug("Skipping move of region " + hri.getRegionNameAsString()<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span>        + " because region already assigned to the same server " + dest + ".");<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span>      return;<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span>    }<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span><a name="line.1983"></a>
+<span class="sourceLineNo">1984</span>    // Now we can do the move<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span>    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), dest);<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span>    assert rp.getDestination() != null: rp.toString() + " " + dest;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span><a name="line.1987"></a>
+<span class="sourceLineNo">1988</span>    try {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span>      checkInitialized();<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span>      if (this.cpHost != null) {<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span>        this.cpHost.preMove(hri, rp.getSource(), rp.getDestination());<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span>      }<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span>      // Warmup the region on the destination before initiating the move. this call<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span>      // is synchronous and takes some time. doing it before the source region gets<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span>      // closed<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span>      serverManager.sendRegionWarmup(rp.getDestination(), hri);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span><a name="line.1997"></a>
+<span class="sourceLineNo">1998</span>      LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span>      Future&lt;byte []&gt; future = this.assignmentManager.moveAsync(rp);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span>      try {<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span>        // Is this going to work? Will we throw exception on error?<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span>        // TODO: CompletableFuture rather than this stunted Future.<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span>        future.get();<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span>      } catch (InterruptedException | ExecutionException e) {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span>        throw new HBaseIOException(e);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span>      }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span>      if (this.cpHost != null) {<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span>        this.cpHost.postMove(hri, rp.getSource(), rp.getDestination());<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span>      }<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span>    } catch (IOException ioe) {<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span>      if (ioe instanceof HBaseIOException) {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span>        throw (HBaseIOException)ioe;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span>      }<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span>      throw new HBaseIOException(ioe);<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span>    }<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span>  }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span>  @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span>  public long createTable(<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span>      final TableDescriptor tableDescriptor,<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span>      final byte [][] splitKeys,<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span>      final long nonceGroup,<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>      final long nonce) throws IOException {<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span>    checkInitialized();<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span><a name="line.2025"></a>
+<span class="sourceLineNo">2026</span>    String namespace = tableDescriptor.getTableName().getNamespaceAsString();<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span>    this.clusterSchemaService.getNamespace(namespace);<a name="line.2027"></a>
+<span class="sourceLineNo">2028</span><a name="line.2028"></a>
+<span class="sourceLineNo">2029</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span>    sanityCheckTableDescriptor(tableDescriptor);<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span><a name="line.2031"></a>
+<span class="sourceLineNo">2032</span>    return MasterProcedureUtil.submitProcedure(<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span>        new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span>      @Override<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span>      protected void run() throws IOException {<a name="line.2035"></a>
+<span class="sourceLineNo">2036</span>        getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);<a name="line.2036"></a>
 <span class="sourceLineNo">2037</span><a name="line.2037"></a>
-<span class="sourceLineNo">2038</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span>        //<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2042"></a>
-<span class="sourceLineNo">2043</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span>        submitProcedure(new CreateTableProcedure(<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span>        latch.await();<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span>      }<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span><a name="line.2050"></a>
-<span class="sourceLineNo">2051</span>      @Override<a name="line.2051"></a>
-<span class="sourceLineNo">2052</span>      protected String getDescription() {<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span>        return "CreateTableProcedure";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span>      }<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span>    });<a name="line.2055"></a>
-<span class="sourceLineNo">2056</span>  }<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span><a name="line.2057"></a>
-<span class="sourceLineNo">2058</span>  @Override<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2059"></a>
-<span class="sourceLineNo">2060</span>    if (isStopped()) {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span>      throw new MasterNotRunningException();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span>    }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span>    if (!(tableName.isSystemTable())) {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span>      throw new IllegalArgumentException(<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span>        "Only system table creation can use this createSystemTable API");<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span>    }<a name="line.2068"></a>
-<span class="sourceLineNo">2069</span><a name="line.2069"></a>
-<span class="sourceLineNo">2070</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2070"></a>
+<span class="sourceLineNo">2038</span>        LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span><a name="line.2039"></a>
+<span class="sourceLineNo">2040</span>        // TODO: We can handle/merge duplicate requests, and differentiate the case of<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span>        //       TableExistsException by saying if the schema is the same or not.<a name="line.2041"></a>
+<span class="sourceLineNo">2042</span>        //<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span>        // We need to wait for the procedure to potentially fail due to "prepare" sanity<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span>        // checks. This will block only the beginning of the procedure. See HBASE-19953.<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span>        ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span>        submitProcedure(new CreateTableProcedure(<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span>            procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span>        latch.await();<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span>        getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span>      }<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span><a name="line.2052"></a>
+<span class="sourceLineNo">2053</span>      @Override<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span>      protected String getDescription() {<a name="line.2054"></a>
+<span class="sourceLineNo">2055</span>        return "CreateTableProcedure";<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span>      }<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span>    });<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span>  }<a name="line.2058"></a>
+<span class="sourceLineNo">2059</span><a name="line.2059"></a>
+<span class="sourceLineNo">2060</span>  @Override<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span>  public long createSystemTable(final TableDescriptor tableDescriptor) throws IOException {<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span>    if (isStopped()) {<a name="line.2062"></a>
+<span class="sourceLineNo">2063</span>      throw new MasterNotRunningException();<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span>    }<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span><a name="line.2065"></a>
+<span class="sourceLineNo">2066</span>    TableName tableName = tableDescriptor.getTableName();<a name="line.2066"></a>
+<span class="sourceLineNo">2067</span>    if (!(tableName.isSystemTable())) {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span>      throw new IllegalArgumentException(<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span>        "Only system table creation can use this createSystemTable API");<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span>    }<a name="line.2070"></a>
 <span class="sourceLineNo">2071</span><a name="line.2071"></a>
-<span class="sourceLineNo">2072</span>    LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);<a name="line.2072"></a>
+<span class="sourceLineNo">2072</span>    RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, null);<a name="line.2072"></a>
 <span class="sourceLineNo">2073</span><a name="line.2073"></a>
-<span class="sourceLineNo">2074</span>    // This special create table is called locally to master.  Therefore, no RPC means no need<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span>    // to use nonce to detect duplicated RPC call.<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span>    long procId = this.procedureExecutor.submitProcedure(<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span>      new CreateTableProcedure(procedureExecutor.getEnvironment(), tableDescriptor, newRegions));<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span><a name="line.2078"></a>
-<span class="sourceLineNo">2079</span>    return procId;<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span>  }<a name="line.2080"></a>
-<span class="sourceLineNo">2081</span><a name="line.2081"></a>
-<span class="sourceLineNo">2082</span>  /**<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span>   * Checks whether the table conforms to some sane limits, and configured<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span>   * values (compression, etc) work. Throws an exception if something is wrong.<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span>   * @throws IOException<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span>   */<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span>  private void sanityCheckTableDescriptor(final TableDescriptor htd) throws IOException {<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span>    final String CONF_KEY = "hbase.table.sanity.checks";<a name="line.2088"></a>
-<span class="sourceLineNo">2089</span>    boolean logWarn = false;<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span>    if (!conf.getBoolean(CONF_KEY, true)) {<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span>      logWarn = true;<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span>    }<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span>    String tableVal = htd.getValue(CONF_KEY);<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span>    if (tableVal != null &amp;&amp; !Boolean.valueOf(tableVal)) {<a name="line.2094"></a>
-<span class="sourceLineNo">2095</span>      logWarn = true;<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span>    }<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span><a name="line.2097"></a>
-<span class="sourceLineNo">2098</span>    // check max file size<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span>    long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span>    long maxFileSize = htd.getMaxFileSize();<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span>    if (maxFileSize &lt; 0) {<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span>      maxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, maxFileSizeLowerLimit);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span>    }<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span>    if (maxFileSize &lt; conf.getLong("hbase.hregion.max.filesize.limit", maxFileSizeLowerLimit)) {<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span>      String message = "MAX_FILESIZE for table descriptor or "<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span>          + "\"hbase.hregion.max.filesize\" (" + maxFileSize<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span>          + ") is too small, which might cause over splitting into unmanageable "<a name="line.2107"></a>
-<span class="sourceLineNo">2108</span>          + "number of regions.";<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span>    }<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span><a name="line.2111"></a>
-<span class="sourceLineNo">2112</span>    // check flush size<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span>    long flushSizeLowerLimit = 1024 * 1024L; // 1M is the default lower limit<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span>    long flushSize = htd.getMemStoreFlushSize();<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span>    if (flushSize &lt; 0) {<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span>      flushSize = conf.getLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSizeLowerLimit);<a name="line.2116"></a>
-<span class="sourceLineNo">2117</span>    }<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span>    if (flushSize &lt; conf.getLong("hbase.hregion.memstore.flush.size.limit", flushSizeLowerLimit)) {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span>      String message = "MEMSTORE_FLUSHSIZE for table descriptor or "<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span>          + "\"hbase.hregion.memstore.flush.size\" ("+flushSize+") is too small, which might cause"<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span>          + " very frequent flushing.";<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2122"></a>
-<span class="sourceLineNo">2123</span>    }<a name="line.2123"></a>
-<span class="sourceLineNo">2124</span><a name="line.2124"></a>
-<span class="sourceLineNo">2125</span>    // check that coprocessors and other specified plugin classes can be loaded<a name="line.2125"></a>
-<span class="sourceLineNo">2126</span>    try {<a name="line.2126"></a>
-<span class="sourceLineNo">2127</span>      checkClassLoading(conf, htd);<a name="line.2127"></a>
-<span class="sourceLineNo">2128</span>    } catch (Exception ex) {<a name="line.2128"></a>
-<span class="sourceLineNo">2129</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, ex.getMessage(), null);<a name="line.2129"></a>
-<span class="sourceLineNo">2130</span>    }<a name="line.2130"></a>
-<span class="sourceLineNo">2131</span><a name="line.2131"></a>
-<span class="sourceLineNo">2132</span>    // check compression can be loaded<a name="line.2132"></a>
-<span class="sourceLineNo">2133</span>    try {<a name="line.2133"></a>
-<span class="sourceLineNo">2134</span>      checkCompression(htd);<a name="line.2134"></a>
-<span class="sourceLineNo">2135</span>    } catch (IOException e) {<a name="line.2135"></a>
-<span class="sourceLineNo">2136</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2136"></a>
-<span class="sourceLineNo">2137</span>    }<a name="line.2137"></a>
-<span class="sourceLineNo">2138</span><a name="line.2138"></a>
-<span class="sourceLineNo">2139</span>    // check encryption can be loaded<a name="line.2139"></a>
-<span class="sourceLineNo">2140</span>    try {<a name="line.2140"></a>
-<span class="sourceLineNo">2141</span>      checkEncryption(conf, htd);<a name="line.2141"></a>
-<span class="sourceLineNo">2142</span>    } catch (IOException e) {<a name="line.2142"></a>
-<span class="sourceLineNo">2143</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, e.getMessage(), e);<a name="line.2143"></a>
-<span class="sourceLineNo">2144</span>    }<a name="line.2144"></a>
-<span class="sourceLineNo">2145</span>    // Verify compaction policy<a name="line.2145"></a>
-<span class="sourceLineNo">2146</span>    try{<a name="line.2146"></a>
-<span class="sourceLineNo">2147</span>      checkCompactionPolicy(conf, htd);<a name="line.2147"></a>
-<span class="sourceLineNo">2148</span>    } catch(IOException e){<a name="line.2148"></a>
-<span class="sourceLineNo">2149</span>      warnOrThrowExceptionForFailure(false, CONF_KEY, e.getMessage(), e);<a name="line.2149"></a>
-<span class="sourceLineNo">2150</span>    }<a name="line.2150"></a>
-<span class="sourceLineNo">2151</span>    // check that we have at least 1 CF<a name="line.2151"></a>
-<span class="sourceLineNo">2152</span>    if (htd.getColumnFamilyCount() == 0) {<a name="line.2152"></a>
-<span class="sourceLineNo">2153</span>      String message = "Table should have at least one column family.";<a name="line.2153"></a>
-<span class="sourceLineNo">2154</span>      warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2154"></a>
-<span class="sourceLineNo">2155</span>    }<a name="line.2155"></a>
-<span class="sourceLineNo">2156</span><a name="line.2156"></a>
-<span class="sourceLineNo">2157</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2157"></a>
-<span class="sourceLineNo">2158</span>      if (hcd.getTimeToLive() &lt;= 0) {<a name="line.2158"></a>
-<span class="sourceLineNo">2159</span>        String message = "TTL for column family " + hcd.getNameAsString() + " must be positive.";<a name="line.2159"></a>
-<span class="sourceLineNo">2160</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2160"></a>
-<span class="sourceLineNo">2161</span>      }<a name="line.2161"></a>
-<span class="sourceLineNo">2162</span><a name="line.2162"></a>
-<span class="sourceLineNo">2163</span>      // check blockSize<a name="line.2163"></a>
-<span class="sourceLineNo">2164</span>      if (hcd.getBlocksize() &lt; 1024 || hcd.getBlocksize() &gt; 16 * 1024 * 1024) {<a name="line.2164"></a>
-<span class="sourceLineNo">2165</span>        String message = "Block size for column family " + hcd.getNameAsString()<a name="line.2165"></a>
-<span class="sourceLineNo">2166</span>            + "  must be between 1K and 16MB.";<a name="line.2166"></a>
-<span class="sourceLineNo">2167</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2167"></a>
-<span class="sourceLineNo">2168</span>      }<a name="line.2168"></a>
-<span class="sourceLineNo">2169</span><a name="line.2169"></a>
-<span class="sourceLineNo">2170</span>      // check versions<a name="line.2170"></a>
-<span class="sourceLineNo">2171</span>      if (hcd.getMinVersions() &lt; 0) {<a name="line.2171"></a>
-<span class="sourceLineNo">2172</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2172"></a>
-<span class="sourceLineNo">2173</span>          + "  must be positive.";<a name="line.2173"></a>
-<span class="sourceLineNo">2174</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2174"></a>
-<span class="sourceLineNo">2175</span>      }<a name="line.2175"></a>
-<span class="sourceLineNo">2176</span>      // max versions already being checked<a name="line.2176"></a>
-<span class="sourceLineNo">2177</span><a name="line.2177"></a>
-<span class="sourceLineNo">2178</span>      // HBASE-13776 Setting illegal versions for ColumnFamilyDescriptor<a name="line.2178"></a>
-<span class="sourceLineNo">2179</span>      //  does not throw IllegalArgumentException<a name="line.2179"></a>
-<span class="sourceLineNo">2180</span>      // check minVersions &lt;= maxVerions<a name="line.2180"></a>
-<span class="sourceLineNo">2181</span>      if (hcd.getMinVersions() &gt; hcd.getMaxVersions()) {<a name="line.2181"></a>
-<span class="sourceLineNo">2182</span>        String message = "Min versions for column family " + hcd.getNameAsString()<a name="line.2182"></a>
-<span class="sourceLineNo">2183</span>            + " must be less than the Max versions.";<a name="line.2183"></a>
-<span class="sourceLineNo">2184</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2184"></a>
-<span class="sourceLineNo">2185</span>      }<a name="line.2185"></a>
-<span class="sourceLineNo">2186</span><a name="line.2186"></a>
-<span class="sourceLineNo">2187</span>      // check replication scope<a name="line.2187"></a>
-<span class="sourceLineNo">2188</span>      checkReplicationScope(hcd);<a name="line.2188"></a>
-<span class="sourceLineNo">2189</span>      // check bloom filter type<a name="line.2189"></a>
-<span class="sourceLineNo">2190</span>      checkBloomFilterType(hcd);<a name="line.2190"></a>
-<span class="sourceLineNo">2191</span><a name="line.2191"></a>
-<span class="sourceLineNo">2192</span>      // check data replication factor, it can be 0(default value) when user has not explicitly<a name="line.2192"></a>
-<span class="sourceLineNo">2193</span>      // set the value, in this case we use default replication factor set in the file system.<a name="line.2193"></a>
-<span class="sourceLineNo">2194</span>      if (hcd.getDFSReplication() &lt; 0) {<a name="line.2194"></a>
-<span class="sourceLineNo">2195</span>        String message = "HFile Replication for column family " + hcd.getNameAsString()<a name="line.2195"></a>
-<span class="sourceLineNo">2196</span>            + "  must be greater than zero.";<a name="line.2196"></a>
-<span class="sourceLineNo">2197</span>        warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null);<a name="line.2197"></a>
-<span class="sourceLineNo">2198</span>      }<a name="line.2198"></a>
-<span class="sourceLineNo">2199</span><a name="line.2199"></a>
-<span class="sourceLineNo">2200</span>      // TODO: should we check coprocessors and encryption ?<a name="line.2200"></a>
-<span class="sourceLineNo">2201</span>    }<a name="line.2201"></a>
-<span class="sourceLineNo">2202</span>  }<a name="line.2202"></a>
-<span class="sourceLineNo">2203</span><a name="line.2203"></a>
-<span class="sourceLineNo">2204</span>  private void checkReplicationScope(ColumnFamilyDescriptor hcd) throws IOException{<a name="line.2204"></a>
-<span class="sourceLineNo">2205</span>    // check replication scope<a name="line.2205"></a>
-<span class="sourceLineNo">2206</span>    WALProtos.ScopeType scop = WALProtos.ScopeType.valueOf(hcd.getScope());<a name="line.2206"></a>
-<span class="sourceLineNo">2207</span>    if (scop == null) {<a name="line.2207"></a>
-<span class="sourceLineNo">2208</span>      String message = "Replication scope for column family "<a name="line.2208"></a>
-<span class="sourceLineNo">2209</span>          + hcd.getNameAsString() + " is " + hcd.getScope() + " which is invalid.";<a name="line.2209"></a>
-<span class="sourceLineNo">2210</span><a name="line.2210"></a>
-<span class="sourceLineNo">2211</span>      LOG.error(message);<a name="line.2211"></a>
-<span class="sourceLineNo">2212</span>      throw new DoNotRetryIOException(message);<a name="line.2212"></a>
-<span class="sourceLineNo">2213</span>    }<a name="line.2213"></a>
-<span class="sourceLineNo">2214</span>  }<a name="line.2214"></a>
-<span class="sourceLineNo">2215</span><a name="line.2215"></a>
-<span class="sourceLineNo">2216</span>  private void checkCompactionPolicy(Configuration conf, TableDescriptor htd)<a name="line.2216"></a>
-<span class="sourceLineNo">2217</span>      throws IOException {<a name="line.2217"></a>
-<span class="sourceLineNo">2218</span>    // FIFO compaction has some requirements<a name="line.2218"></a>
-<span class="sourceLineNo">2219</span>    // Actually FCP ignores periodic major compactions<a name="line.2219"></a>
-<span class="sourceLineNo">2220</span>    String className = htd.getValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2220"></a>
-<span class="sourceLineNo">2221</span>    if (className == null) {<a name="line.2221"></a>
-<span class="sourceLineNo">2222</span>      className =<a name="line.2222"></a>
-<span class="sourceLineNo">2223</span>          conf.get(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,<a name="line.2223"></a>
-<span class="sourceLineNo">2224</span>            ExploringCompactionPolicy.class.getName());<a name="line.2224"></a>
-<span class="sourceLineNo">2225</span>    }<a name="line.2225"></a>
-<span class="sourceLineNo">2226</span><a name="line.2226"></a>
-<span class="sourceLineNo">2227</span>    int blockingFileCount = HStore.DEFAULT_BLOCKING_STOREFILE_COUNT;<a name="line.2227"></a>
-<span class="sourceLineNo">2228</span>    String sv = htd.getValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2228"></a>
-<span class="sourceLineNo">2229</span>    if (sv != null) {<a name="line.2229"></a>
-<span class="sourceLineNo">2230</span>      blockingFileCount = Integer.parseInt(sv);<a name="line.2230"></a>
-<span class="sourceLineNo">2231</span>    } else {<a name="line.2231"></a>
-<span class="sourceLineNo">2232</span>      blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, blockingFileCount);<a name="line.2232"></a>
-<span class="sourceLineNo">2233</span>    }<a name="line.2233"></a>
-<span class="sourceLineNo">2234</span><a name="line.2234"></a>
-<span class="sourceLineNo">2235</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2235"></a>
-<span class="sourceLineNo">2236</span>      String compactionPolicy =<a name="line.2236"></a>
-<span class="sourceLineNo">2237</span>          hcd.getConfigurationValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY);<a name="line.2237"></a>
-<span class="sourceLineNo">2238</span>      if (compactionPolicy == null) {<a name="line.2238"></a>
-<span class="sourceLineNo">2239</span>        compactionPolicy = className;<a name="line.2239"></a>
-<span class="sourceLineNo">2240</span>      }<a name="line.2240"></a>
-<span class="sourceLineNo">2241</span>      if (!compactionPolicy.equals(FIFOCompactionPolicy.class.getName())) {<a name="line.2241"></a>
-<span class="sourceLineNo">2242</span>        continue;<a name="line.2242"></a>
-<span class="sourceLineNo">2243</span>      }<a name="line.2243"></a>
-<span class="sourceLineNo">2244</span>      // FIFOCompaction<a name="line.2244"></a>
-<span class="sourceLineNo">2245</span>      String message = null;<a name="line.2245"></a>
-<span class="sourceLineNo">2246</span><a name="line.2246"></a>
-<span class="sourceLineNo">2247</span>      // 1. Check TTL<a name="line.2247"></a>
-<span class="sourceLineNo">2248</span>      if (hcd.getTimeToLive() == ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {<a name="line.2248"></a>
-<span class="sourceLineNo">2249</span>        message = "Default TTL is not supported for FIFO compaction";<a name="line.2249"></a>
-<span class="sourceLineNo">2250</span>        throw new IOException(message);<a name="line.2250"></a>
-<span class="sourceLineNo">2251</span>      }<a name="line.2251"></a>
-<span class="sourceLineNo">2252</span><a name="line.2252"></a>
-<span class="sourceLineNo">2253</span>      // 2. Check min versions<a name="line.2253"></a>
-<span class="sourceLineNo">2254</span>      if (hcd.getMinVersions() &gt; 0) {<a name="line.2254"></a>
-<span class="sourceLineNo">2255</span>        message = "MIN_VERSION &gt; 0 is not supported for FIFO compaction";<a name="line.2255"></a>
-<span class="sourceLineNo">2256</span>        throw new IOException(message);<a name="line.2256"></a>
-<span class="sourceLineNo">2257</span>      }<a name="line.2257"></a>
-<span class="sourceLineNo">2258</span><a name="line.2258"></a>
-<span class="sourceLineNo">2259</span>      // 3. blocking file count<a name="line.2259"></a>
-<span class="sourceLineNo">2260</span>      sv = hcd.getConfigurationValue(HStore.BLOCKING_STOREFILES_KEY);<a name="line.2260"></a>
-<span class="sourceLineNo">2261</span>      if (sv != null) {<a name="line.2261"></a>
-<span class="sourceLineNo">2262</span>        blockingFileCount = Integer.parseInt(sv);<a name="line.2262"></a>
-<span class="sourceLineNo">2263</span>      }<a name="line.2263"></a>
-<span class="sourceLineNo">2264</span>      if (blockingFileCount &lt; 1000) {<a name="line.2264"></a>
-<span class="sourceLineNo">2265</span>        message =<a name="line.2265"></a>
-<span class="sourceLineNo">2266</span>            "Blocking file count '" + HStore.BLOCKING_STOREFILES_KEY + "' " + blockingFileCount<a name="line.2266"></a>
-<span class="sourceLineNo">2267</span>                + " is below recommended minimum of 1000 for column family "+ hcd.getNameAsString();<a name="line.2267"></a>
-<span class="sourceLineNo">2268</span>        throw new IOException(message);<a name="line.2268"></a>
-<span class="sourceLineNo">2269</span>      }<a name="line.2269"></a>
-<span class="sourceLineNo">2270</span>    }<a name="line.2270"></a>
-<span class="sourceLineNo">2271</span>  }<a name="line.2271"></a>
-<span class="sourceLineNo">2272</span><a name="line.2272"></a>
-<span class="sourceLineNo">2273</span>  private static void checkBloomFilterType(ColumnFamilyDescriptor cfd)<a name="line.2273"></a>
-<span class="sourceLineNo">2274</span>      throws IOException {<a name="line.2274"></a>
-<span class="sourceLineNo">2275</span>    Configuration conf = new CompoundConfiguration().addStringMap(cfd.getConfiguration());<a name="line.2275"></a>
-<span class="sourceLineNo">2276</span>    try {<a name="line.2276"></a>
-<span class="sourceLineNo">2277</span>      BloomFilterUtil.getBloomFilterParam(cfd.getBloomFilterType(), conf);<a name="line.2277"></a>
-<span class="sourceLineNo">2278</span>    } catch (IllegalArgumentException e) {<a name="line.2278"></a>
-<span class="sourceLineNo">2279</span>      throw new DoNotRetryIOException("Failed to get bloom filter param", e);<a name="line.2279"></a>
-<span class="sourceLineNo">2280</span>    }<a name="line.2280"></a>
-<span class="sourceLineNo">2281</span>  }<a name="line.2281"></a>
-<span class="sourceLineNo">2282</span><a name="line.2282"></a>
-<span class="sourceLineNo">2283</span>  // HBASE-13350 - Helper method to log warning on sanity check failures if checks disabled.<a name="line.2283"></a>
-<span class="sourceLineNo">2284</span>  private static void warnOrThrowExceptionForFailure(boolean logWarn, String confKey,<a name="line.2284"></a>
-<span class="sourceLineNo">2285</span>      String message, Exception cause) throws IOException {<a name="line.2285"></a>
-<span class="sourceLineNo">2286</span>    if (!logWarn) {<a name="line.2286"></a>
-<span class="sourceLineNo">2287</span>      throw new DoNotRetryIOException(message + " Set " + confKey +<a name="line.2287"></a>
-<span class="sourceLineNo">2288</span>          " to false at conf or table descriptor if you want to bypass sanity checks", cause);<a name="line.2288"></a>
-<span class="sourceLineNo">2289</span>    }<a name="line.2289"></a>
-<span class="sourceLineNo">2290</span>    LOG.warn(message);<a name="line.2290"></a>
-<span class="sourceLineNo">2291</span>  }<a name="line.2291"></a>
-<span class="sourceLineNo">2292</span><a name="line.2292"></a>
-<span class="sourceLineNo">2293</span>  private void startActiveMasterManager(int infoPort) throws KeeperException {<a name="line.2293"></a>
-<span class="sourceLineNo">2294</span>    String backupZNode = ZNodePaths.joinZNode(<a name="line.2294"></a>
-<span class="sourceLineNo">2295</span>      zooKeeper.getZNodePaths().backupMasterAddressesZNode, serverName.toString());<a name="line.2295"></a>
-<span class="sourceLineNo">2296</span>    /*<a name="line.2296"></a>
-<span class="sourceLineNo">2297</span>    * Add a ZNode for ourselves in the backup master directory since we<a name="line.2297"></a>
-<span class="sourceLineNo">2298</span>    * may not become the active master. If so, we want the actual active<a name="line.2298"></a>
-<span class="sourceLineNo">2299</span>    * master to know we are backup masters, so that it won't assign<a name="line.2299"></a>
-<span class="sourceLineNo">2300</span>    * regions to us if so configured.<a name="line.2300"></a>
-<span class="sourceLineNo">2301</span>    *<a name="line.2301"></a>
-<span class="sourceLineNo">2302</span>    * If we become the active master later, ActiveMasterManager will delete<a name="line.2302"></a>
-<span class="sourceLineNo">2303</span>    * this node explicitly.  If we crash before then, ZooKeeper will delete<a name="line.2303"></a>
-<span class="sourceLineNo">2304</span>    * this node for us since it is ephemeral.<a name="line.2304"></a>
-<span class="sourceLineNo">2305</span>    */<a name="line.2305"></a>
-<span class="sourceLineNo">2306</span>    LOG.info("Adding backup master ZNode " + backupZNode);<a name="line.2306"></a>
-<span class="sourceLineNo">2307</span>    if (!MasterAddressTracker.setMasterAddress(zooKeeper, backupZNode, serverName, infoPort)) {<a name="line.2307"></a>
-<span class="sourceLineNo">2308</span>      LOG.warn("Failed create of " + backupZNode + " by " + serverName);<a name="line.2308"></a>
-<span class="sourceLineNo">2309</span>    }<a name="line.2309"></a>
-<span class="sourceLineNo">2310</span>    this.activeMasterManager.setInfoPort(infoPort);<a name="line.2310"></a>
-<span class="sourceLineNo">2311</span>    int timeout = conf.getInt(HConstants.ZK_SESSION_TIMEOUT, HConstants.DEFAULT_ZK_SESSION_TIMEOUT);<a name="line.2311"></a>
-<span class="sourceLineNo">2312</span>    // If we're a backup master, stall until a primary to write this address<a name="line.2312"></a>
-<span class="sourceLineNo">2313</span>    if (conf.getBoolean(HConstants.MASTER_TYPE_BACKUP, HConstants.DEFAULT_MASTER_TYPE_BACKUP)) {<a name="line.2313"></a>
-<span class="sourceLineNo">2314</span>      LOG.debug("HMaster started in backup mode. Stalling until master znode is written.");<a name="line.2314"></a>
-<span class="sourceLineNo">2315</span>      // This will only be a minute or so while the cluster starts up,<a name="line.2315"></a>
-<span class="sourceLineNo">2316</span>      // so don't worry about setting watches on the parent znode<a name="line.2316"></a>
-<span class="sourceLineNo">2317</span>      while (!activeMasterManager.hasActiveMaster()) {<a name="line.2317"></a>
-<span class="sourceLineNo">2318</span>        LOG.debug("Waiting for master address and cluster state znode to be written.");<a name="line.2318"></a>
-<span class="sourceLineNo">2319</span>        Threads.sleep(timeout);<a name="line.2319"></a>
-<span class="sourceLineNo">2320</span>      }<a name="line.2320"></a>
-<span class="sourceLineNo">2321</span>    }<a name="line.2321"></a>
-<span class="sourceLineNo">2322</span>    MonitoredTask status = TaskMonitor.get().createStatus("Master startup");<a name="line.2322"></a>
-<span class="sourceLineNo">2323</span>    status.setDescription("Master startup");<a name="line.2323"></a>
-<span class="sourceLineNo">2324</span>    try {<a name="line.2324"></a>
-<span class="sourceLineNo">2325</span>      if (activeMasterManager.blockUntilBecomingActiveMaster(timeout, status)) {<a name="line.2325"></a>
-<span class="sourceLineNo">2326</span>        finishActiveMasterInitialization(status);<a name="line.2326"></a>
-<span class="sourceLineNo">2327</span>      }<a name="line.2327"></a>
-<span class="sourceLineNo">2328</span>    } catch (Throwable t) {<a name="line.2328"></a>
-<span class="sourceLineNo">2329</span>      status.setStatus("Failed to become active: " + t.getMessage());<a name="line.2329"></a>
-<span class="sourceLineNo">2330</span>      LOG.error(HBaseMarkers.FATAL, "Failed to become active master", t);<a name="line.2330"></a>
-<span class="sourceLineNo">2331</span>      // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility<a name="line.2331"></a>
-<span class="sourceLineNo">2332</span>      if (t instanceof NoClassDefFoundError &amp;&amp; t.getMessage().<a name="line.2332"></a>
-<span class="sourceLineNo">2333</span>          contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) {<a name="line.2333"></a>
-<span class="sourceLineNo">2334</span>        // improved error message for this special case<a name="line.2334"></a>
-<span class="sourceLineNo">2335</span>        abort("HBase is having a problem with its Hadoop jars.  You may need to recompile " +<a name="line.2335"></a>
-<span class="sourceLineNo">2336</span>          "HBase against Hadoop version " + org.apache.hadoop.util.VersionInfo.getVersion() +<a name="line.2336"></a>
-<span class="sourceLineNo">2337</span>          " or change your hadoop jars to start properly", t);<a name="line.2337"></a>
-<span class="sourceLineNo">2338</span>      } else {<a name="line.2338"></a>
-<span class="sourceLineNo">2339</span>        abort("Unhandled exception. Starting shutdown.", t);<a name="line.2339"></a>
-<span class="sourceLineNo">2340</span>      }<a name="line.2340"></a>
-<span class="sourceLineNo">2341</span>    } finally {<a name="line.2341"></a>
-<span class="sourceLineNo">2342</span>      status.cleanup();<a name="line.2342"></a>
-<span class="sourceLineNo">2343</span>    }<a name="line.2343"></a>
-<span class="sourceLineNo">2344</span>  }<a name="line.2344"></a>
-<span class="sourceLineNo">2345</span><a name="line.2345"></a>
-<span class="sourceLineNo">2346</span>  private void checkCompression(final TableDescriptor htd)<a name="line.2346"></a>
-<span class="sourceLineNo">2347</span>  throws IOException {<a name="line.2347"></a>
-<span class="sourceLineNo">2348</span>    if (!this.masterCheckCompression) return;<a name="line.2348"></a>
-<span class="sourceLineNo">2349</span>    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {<a name="line.2349"></a>
-<span class="sourceLineNo">2350</span>      checkCompression(hcd);<a name="line.2350"></a>
-<span class="sourceLineNo">2351</span>    }<a name="line.2351"></a>
-<span class="sourceLineNo">2352</span>  }<a name="line.2352"></a>
-<span class="sourceLineNo">2353</span><a name="line.2353"></a>
-<span class="sourceLineNo">2354</span>  private void checkCompression(final ColumnFamilyDescriptor hcd)<a name="line.2354"></a>
-<span class="sourceLineNo">2355</span>  throws IOException {<a name="line.2355"></a>
-<span class="sourceLineNo">2356</span>    if (!this.masterCheckCompression) return;<a name="line.2356"></a>
-<span class="sourceLineNo">2357</span>    CompressionTest.testCompression(hcd.getCompressionType());<a name="line.2357"></a>
-<span class="sourceLineNo">2358</span>    CompressionTest.testCompression(hcd.getCompactionCompressionType());<a name="line.2358"><

<TRUNCATED>

[12/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
index adff387..71006ea 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html
@@ -32,185 +32,190 @@
 <span class="sourceLineNo">024</span>import java.util.Set;<a name="line.24"></a>
 <span class="sourceLineNo">025</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.25"></a>
 <span class="sourceLineNo">026</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.26"></a>
-<span class="sourceLineNo">027</span><a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.28"></a>
+<span class="sourceLineNo">027</span>import org.slf4j.Logger;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.slf4j.LoggerFactory;<a name="line.28"></a>
 <span class="sourceLineNo">029</span><a name="line.29"></a>
-<span class="sourceLineNo">030</span>/**<a name="line.30"></a>
-<span class="sourceLineNo">031</span> * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".<a name="line.31"></a>
-<span class="sourceLineNo">032</span> * A "Root Procedure" is a Procedure without parent, each subprocedure will be<a name="line.32"></a>
-<span class="sourceLineNo">033</span> * added to the "Root Procedure" stack (or rollback-stack).<a name="line.33"></a>
-<span class="sourceLineNo">034</span> *<a name="line.34"></a>
-<span class="sourceLineNo">035</span> * RootProcedureState is used and managed only by the ProcedureExecutor.<a name="line.35"></a>
-<span class="sourceLineNo">036</span> *    Long rootProcId = getRootProcedureId(proc);<a name="line.36"></a>
-<span class="sourceLineNo">037</span> *    rollbackStack.get(rootProcId).acquire(proc)<a name="line.37"></a>
-<span class="sourceLineNo">038</span> *    rollbackStack.get(rootProcId).release(proc)<a name="line.38"></a>
-<span class="sourceLineNo">039</span> *    ...<a name="line.39"></a>
-<span class="sourceLineNo">040</span> */<a name="line.40"></a>
-<span class="sourceLineNo">041</span>@InterfaceAudience.Private<a name="line.41"></a>
-<span class="sourceLineNo">042</span>@InterfaceStability.Evolving<a name="line.42"></a>
-<span class="sourceLineNo">043</span>class RootProcedureState&lt;TEnvironment&gt; {<a name="line.43"></a>
-<span class="sourceLineNo">044</span><a name="line.44"></a>
-<span class="sourceLineNo">045</span>  private enum State {<a name="line.45"></a>
-<span class="sourceLineNo">046</span>    RUNNING,         // The Procedure is running or ready to run<a name="line.46"></a>
-<span class="sourceLineNo">047</span>    FAILED,          // The Procedure failed, waiting for the rollback executing<a name="line.47"></a>
-<span class="sourceLineNo">048</span>    ROLLINGBACK,     // The Procedure failed and the execution was rolledback<a name="line.48"></a>
-<span class="sourceLineNo">049</span>  }<a name="line.49"></a>
-<span class="sourceLineNo">050</span><a name="line.50"></a>
-<span class="sourceLineNo">051</span>  private Set&lt;Procedure&lt;TEnvironment&gt;&gt; subprocs = null;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>  private ArrayList&lt;Procedure&lt;TEnvironment&gt;&gt; subprocStack = null;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>  private State state = State.RUNNING;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>  private int running = 0;<a name="line.54"></a>
-<span class="sourceLineNo">055</span><a name="line.55"></a>
-<span class="sourceLineNo">056</span>  public synchronized boolean isFailed() {<a name="line.56"></a>
-<span class="sourceLineNo">057</span>    switch (state) {<a name="line.57"></a>
-<span class="sourceLineNo">058</span>      case ROLLINGBACK:<a name="line.58"></a>
-<span class="sourceLineNo">059</span>      case FAILED:<a name="line.59"></a>
-<span class="sourceLineNo">060</span>        return true;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>      default:<a name="line.61"></a>
-<span class="sourceLineNo">062</span>        break;<a name="line.62"></a>
-<span class="sourceLineNo">063</span>    }<a name="line.63"></a>
-<span class="sourceLineNo">064</span>    return false;<a name="line.64"></a>
-<span class="sourceLineNo">065</span>  }<a name="line.65"></a>
-<span class="sourceLineNo">066</span><a name="line.66"></a>
-<span class="sourceLineNo">067</span>  public synchronized boolean isRollingback() {<a name="line.67"></a>
-<span class="sourceLineNo">068</span>    return state == State.ROLLINGBACK;<a name="line.68"></a>
+<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>/**<a name="line.32"></a>
+<span class="sourceLineNo">033</span> * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".<a name="line.33"></a>
+<span class="sourceLineNo">034</span> * A "Root Procedure" is a Procedure without parent, each subprocedure will be<a name="line.34"></a>
+<span class="sourceLineNo">035</span> * added to the "Root Procedure" stack (or rollback-stack).<a name="line.35"></a>
+<span class="sourceLineNo">036</span> *<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * RootProcedureState is used and managed only by the ProcedureExecutor.<a name="line.37"></a>
+<span class="sourceLineNo">038</span> *    Long rootProcId = getRootProcedureId(proc);<a name="line.38"></a>
+<span class="sourceLineNo">039</span> *    rollbackStack.get(rootProcId).acquire(proc)<a name="line.39"></a>
+<span class="sourceLineNo">040</span> *    rollbackStack.get(rootProcId).release(proc)<a name="line.40"></a>
+<span class="sourceLineNo">041</span> *    ...<a name="line.41"></a>
+<span class="sourceLineNo">042</span> */<a name="line.42"></a>
+<span class="sourceLineNo">043</span>@InterfaceAudience.Private<a name="line.43"></a>
+<span class="sourceLineNo">044</span>@InterfaceStability.Evolving<a name="line.44"></a>
+<span class="sourceLineNo">045</span>class RootProcedureState&lt;TEnvironment&gt; {<a name="line.45"></a>
+<span class="sourceLineNo">046</span><a name="line.46"></a>
+<span class="sourceLineNo">047</span>  private static final Logger LOG = LoggerFactory.getLogger(RootProcedureState.class);<a name="line.47"></a>
+<span class="sourceLineNo">048</span><a name="line.48"></a>
+<span class="sourceLineNo">049</span>  private enum State {<a name="line.49"></a>
+<span class="sourceLineNo">050</span>    RUNNING,         // The Procedure is running or ready to run<a name="line.50"></a>
+<span class="sourceLineNo">051</span>    FAILED,          // The Procedure failed, waiting for the rollback executing<a name="line.51"></a>
+<span class="sourceLineNo">052</span>    ROLLINGBACK,     // The Procedure failed and the execution was rolledback<a name="line.52"></a>
+<span class="sourceLineNo">053</span>  }<a name="line.53"></a>
+<span class="sourceLineNo">054</span><a name="line.54"></a>
+<span class="sourceLineNo">055</span>  private Set&lt;Procedure&lt;TEnvironment&gt;&gt; subprocs = null;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>  private ArrayList&lt;Procedure&lt;TEnvironment&gt;&gt; subprocStack = null;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>  private State state = State.RUNNING;<a name="line.57"></a>
+<span class="sourceLineNo">058</span>  private int running = 0;<a name="line.58"></a>
+<span class="sourceLineNo">059</span><a name="line.59"></a>
+<span class="sourceLineNo">060</span>  public synchronized boolean isFailed() {<a name="line.60"></a>
+<span class="sourceLineNo">061</span>    switch (state) {<a name="line.61"></a>
+<span class="sourceLineNo">062</span>      case ROLLINGBACK:<a name="line.62"></a>
+<span class="sourceLineNo">063</span>      case FAILED:<a name="line.63"></a>
+<span class="sourceLineNo">064</span>        return true;<a name="line.64"></a>
+<span class="sourceLineNo">065</span>      default:<a name="line.65"></a>
+<span class="sourceLineNo">066</span>        break;<a name="line.66"></a>
+<span class="sourceLineNo">067</span>    }<a name="line.67"></a>
+<span class="sourceLineNo">068</span>    return false;<a name="line.68"></a>
 <span class="sourceLineNo">069</span>  }<a name="line.69"></a>
 <span class="sourceLineNo">070</span><a name="line.70"></a>
-<span class="sourceLineNo">071</span>  /**<a name="line.71"></a>
-<span class="sourceLineNo">072</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.72"></a>
-<span class="sourceLineNo">073</span>   */<a name="line.73"></a>
-<span class="sourceLineNo">074</span>  protected synchronized boolean setRollback() {<a name="line.74"></a>
-<span class="sourceLineNo">075</span>    if (running == 0 &amp;&amp; state == State.FAILED) {<a name="line.75"></a>
-<span class="sourceLineNo">076</span>      state = State.ROLLINGBACK;<a name="line.76"></a>
-<span class="sourceLineNo">077</span>      return true;<a name="line.77"></a>
-<span class="sourceLineNo">078</span>    }<a name="line.78"></a>
-<span class="sourceLineNo">079</span>    return false;<a name="line.79"></a>
-<span class="sourceLineNo">080</span>  }<a name="line.80"></a>
-<span class="sourceLineNo">081</span><a name="line.81"></a>
-<span class="sourceLineNo">082</span>  /**<a name="line.82"></a>
-<span class="sourceLineNo">083</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.83"></a>
-<span class="sourceLineNo">084</span>   */<a name="line.84"></a>
-<span class="sourceLineNo">085</span>  protected synchronized void unsetRollback() {<a name="line.85"></a>
-<span class="sourceLineNo">086</span>    assert state == State.ROLLINGBACK;<a name="line.86"></a>
-<span class="sourceLineNo">087</span>    state = State.FAILED;<a name="line.87"></a>
-<span class="sourceLineNo">088</span>  }<a name="line.88"></a>
-<span class="sourceLineNo">089</span><a name="line.89"></a>
-<span class="sourceLineNo">090</span>  protected synchronized long[] getSubprocedureIds() {<a name="line.90"></a>
-<span class="sourceLineNo">091</span>    if (subprocs == null) {<a name="line.91"></a>
-<span class="sourceLineNo">092</span>      return null;<a name="line.92"></a>
-<span class="sourceLineNo">093</span>    }<a name="line.93"></a>
-<span class="sourceLineNo">094</span>    return subprocs.stream().mapToLong(Procedure::getProcId).toArray();<a name="line.94"></a>
-<span class="sourceLineNo">095</span>  }<a name="line.95"></a>
-<span class="sourceLineNo">096</span><a name="line.96"></a>
-<span class="sourceLineNo">097</span>  protected synchronized List&lt;Procedure&lt;TEnvironment&gt;&gt; getSubproceduresStack() {<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    return subprocStack;<a name="line.98"></a>
+<span class="sourceLineNo">071</span>  public synchronized boolean isRollingback() {<a name="line.71"></a>
+<span class="sourceLineNo">072</span>    return state == State.ROLLINGBACK;<a name="line.72"></a>
+<span class="sourceLineNo">073</span>  }<a name="line.73"></a>
+<span class="sourceLineNo">074</span><a name="line.74"></a>
+<span class="sourceLineNo">075</span>  /**<a name="line.75"></a>
+<span class="sourceLineNo">076</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.76"></a>
+<span class="sourceLineNo">077</span>   */<a name="line.77"></a>
+<span class="sourceLineNo">078</span>  protected synchronized boolean setRollback() {<a name="line.78"></a>
+<span class="sourceLineNo">079</span>    if (running == 0 &amp;&amp; state == State.FAILED) {<a name="line.79"></a>
+<span class="sourceLineNo">080</span>      state = State.ROLLINGBACK;<a name="line.80"></a>
+<span class="sourceLineNo">081</span>      return true;<a name="line.81"></a>
+<span class="sourceLineNo">082</span>    }<a name="line.82"></a>
+<span class="sourceLineNo">083</span>    return false;<a name="line.83"></a>
+<span class="sourceLineNo">084</span>  }<a name="line.84"></a>
+<span class="sourceLineNo">085</span><a name="line.85"></a>
+<span class="sourceLineNo">086</span>  /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.87"></a>
+<span class="sourceLineNo">088</span>   */<a name="line.88"></a>
+<span class="sourceLineNo">089</span>  protected synchronized void unsetRollback() {<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    assert state == State.ROLLINGBACK;<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    state = State.FAILED;<a name="line.91"></a>
+<span class="sourceLineNo">092</span>  }<a name="line.92"></a>
+<span class="sourceLineNo">093</span><a name="line.93"></a>
+<span class="sourceLineNo">094</span>  protected synchronized long[] getSubprocedureIds() {<a name="line.94"></a>
+<span class="sourceLineNo">095</span>    if (subprocs == null) {<a name="line.95"></a>
+<span class="sourceLineNo">096</span>      return null;<a name="line.96"></a>
+<span class="sourceLineNo">097</span>    }<a name="line.97"></a>
+<span class="sourceLineNo">098</span>    return subprocs.stream().mapToLong(Procedure::getProcId).toArray();<a name="line.98"></a>
 <span class="sourceLineNo">099</span>  }<a name="line.99"></a>
 <span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span>  protected synchronized RemoteProcedureException getException() {<a name="line.101"></a>
-<span class="sourceLineNo">102</span>    if (subprocStack != null) {<a name="line.102"></a>
-<span class="sourceLineNo">103</span>      for (Procedure&lt;TEnvironment&gt; proc: subprocStack) {<a name="line.103"></a>
-<span class="sourceLineNo">104</span>        if (proc.hasException()) {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>          return proc.getException();<a name="line.105"></a>
-<span class="sourceLineNo">106</span>        }<a name="line.106"></a>
-<span class="sourceLineNo">107</span>      }<a name="line.107"></a>
-<span class="sourceLineNo">108</span>    }<a name="line.108"></a>
-<span class="sourceLineNo">109</span>    return null;<a name="line.109"></a>
-<span class="sourceLineNo">110</span>  }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span>  /**<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * Called by the ProcedureExecutor to mark the procedure step as running.<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   */<a name="line.114"></a>
-<span class="sourceLineNo">115</span>  protected synchronized boolean acquire(Procedure&lt;TEnvironment&gt; proc) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span>    if (state != State.RUNNING) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>      return false;<a name="line.117"></a>
-<span class="sourceLineNo">118</span>    }<a name="line.118"></a>
-<span class="sourceLineNo">119</span><a name="line.119"></a>
-<span class="sourceLineNo">120</span>    running++;<a name="line.120"></a>
-<span class="sourceLineNo">121</span>    return true;<a name="line.121"></a>
-<span class="sourceLineNo">122</span>  }<a name="line.122"></a>
+<span class="sourceLineNo">101</span>  protected synchronized List&lt;Procedure&lt;TEnvironment&gt;&gt; getSubproceduresStack() {<a name="line.101"></a>
+<span class="sourceLineNo">102</span>    return subprocStack;<a name="line.102"></a>
+<span class="sourceLineNo">103</span>  }<a name="line.103"></a>
+<span class="sourceLineNo">104</span><a name="line.104"></a>
+<span class="sourceLineNo">105</span>  protected synchronized RemoteProcedureException getException() {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>    if (subprocStack != null) {<a name="line.106"></a>
+<span class="sourceLineNo">107</span>      for (Procedure&lt;TEnvironment&gt; proc: subprocStack) {<a name="line.107"></a>
+<span class="sourceLineNo">108</span>        if (proc.hasException()) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span>          return proc.getException();<a name="line.109"></a>
+<span class="sourceLineNo">110</span>        }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>      }<a name="line.111"></a>
+<span class="sourceLineNo">112</span>    }<a name="line.112"></a>
+<span class="sourceLineNo">113</span>    return null;<a name="line.113"></a>
+<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
+<span class="sourceLineNo">115</span><a name="line.115"></a>
+<span class="sourceLineNo">116</span>  /**<a name="line.116"></a>
+<span class="sourceLineNo">117</span>   * Called by the ProcedureExecutor to mark the procedure step as running.<a name="line.117"></a>
+<span class="sourceLineNo">118</span>   */<a name="line.118"></a>
+<span class="sourceLineNo">119</span>  protected synchronized boolean acquire(Procedure&lt;TEnvironment&gt; proc) {<a name="line.119"></a>
+<span class="sourceLineNo">120</span>    if (state != State.RUNNING) {<a name="line.120"></a>
+<span class="sourceLineNo">121</span>      return false;<a name="line.121"></a>
+<span class="sourceLineNo">122</span>    }<a name="line.122"></a>
 <span class="sourceLineNo">123</span><a name="line.123"></a>
-<span class="sourceLineNo">124</span>  /**<a name="line.124"></a>
-<span class="sourceLineNo">125</span>   * Called by the ProcedureExecutor to mark the procedure step as finished.<a name="line.125"></a>
-<span class="sourceLineNo">126</span>   */<a name="line.126"></a>
-<span class="sourceLineNo">127</span>  protected synchronized void release(Procedure&lt;TEnvironment&gt; proc) {<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    running--;<a name="line.128"></a>
-<span class="sourceLineNo">129</span>  }<a name="line.129"></a>
-<span class="sourceLineNo">130</span><a name="line.130"></a>
-<span class="sourceLineNo">131</span>  protected synchronized void abort() {<a name="line.131"></a>
-<span class="sourceLineNo">132</span>    if (state == State.RUNNING) {<a name="line.132"></a>
-<span class="sourceLineNo">133</span>      state = State.FAILED;<a name="line.133"></a>
-<span class="sourceLineNo">134</span>    }<a name="line.134"></a>
-<span class="sourceLineNo">135</span>  }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span>  /**<a name="line.137"></a>
-<span class="sourceLineNo">138</span>   * Called by the ProcedureExecutor after the procedure step is completed,<a name="line.138"></a>
-<span class="sourceLineNo">139</span>   * to add the step to the rollback list (or procedure stack)<a name="line.139"></a>
-<span class="sourceLineNo">140</span>   */<a name="line.140"></a>
-<span class="sourceLineNo">141</span>  protected synchronized void addRollbackStep(Procedure&lt;TEnvironment&gt; proc) {<a name="line.141"></a>
-<span class="sourceLineNo">142</span>    if (proc.isFailed()) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>      state = State.FAILED;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    }<a name="line.144"></a>
-<span class="sourceLineNo">145</span>    if (subprocStack == null) {<a name="line.145"></a>
-<span class="sourceLineNo">146</span>      subprocStack = new ArrayList&lt;&gt;();<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    }<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    proc.addStackIndex(subprocStack.size());<a name="line.148"></a>
-<span class="sourceLineNo">149</span>    subprocStack.add(proc);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span>  protected synchronized void addSubProcedure(Procedure&lt;TEnvironment&gt; proc) {<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    if (!proc.hasParent()) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>      return;<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    }<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    if (subprocs == null) {<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      subprocs = new HashSet&lt;&gt;();<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
-<span class="sourceLineNo">159</span>    subprocs.add(proc);<a name="line.159"></a>
-<span class="sourceLineNo">160</span>  }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span>  /**<a name="line.162"></a>
-<span class="sourceLineNo">163</span>   * Called on store load by the ProcedureExecutor to load part of the stack.<a name="line.163"></a>
-<span class="sourceLineNo">164</span>   *<a name="line.164"></a>
-<span class="sourceLineNo">165</span>   * Each procedure has its own stack-positions. Which means we have to write<a name="line.165"></a>
-<span class="sourceLineNo">166</span>   * to the store only the Procedure we executed, and nothing else.<a name="line.166"></a>
-<span class="sourceLineNo">167</span>   * on load we recreate the full stack by aggregating each procedure stack-positions.<a name="line.167"></a>
-<span class="sourceLineNo">168</span>   */<a name="line.168"></a>
-<span class="sourceLineNo">169</span>  protected synchronized void loadStack(Procedure&lt;TEnvironment&gt; proc) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    addSubProcedure(proc);<a name="line.170"></a>
-<span class="sourceLineNo">171</span>    int[] stackIndexes = proc.getStackIndexes();<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    if (stackIndexes != null) {<a name="line.172"></a>
-<span class="sourceLineNo">173</span>      if (subprocStack == null) {<a name="line.173"></a>
-<span class="sourceLineNo">174</span>        subprocStack = new ArrayList&lt;&gt;();<a name="line.174"></a>
-<span class="sourceLineNo">175</span>      }<a name="line.175"></a>
-<span class="sourceLineNo">176</span>      int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>      if (diff &gt; 0) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>        subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);<a name="line.178"></a>
-<span class="sourceLineNo">179</span>        while (diff-- &gt; 0) subprocStack.add(null);<a name="line.179"></a>
+<span class="sourceLineNo">124</span>    running++;<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    return true;<a name="line.125"></a>
+<span class="sourceLineNo">126</span>  }<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>  /**<a name="line.128"></a>
+<span class="sourceLineNo">129</span>   * Called by the ProcedureExecutor to mark the procedure step as finished.<a name="line.129"></a>
+<span class="sourceLineNo">130</span>   */<a name="line.130"></a>
+<span class="sourceLineNo">131</span>  protected synchronized void release(Procedure&lt;TEnvironment&gt; proc) {<a name="line.131"></a>
+<span class="sourceLineNo">132</span>    running--;<a name="line.132"></a>
+<span class="sourceLineNo">133</span>  }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span>  protected synchronized void abort() {<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    if (state == State.RUNNING) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span>      state = State.FAILED;<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    }<a name="line.138"></a>
+<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span>  /**<a name="line.141"></a>
+<span class="sourceLineNo">142</span>   * Called by the ProcedureExecutor after the procedure step is completed,<a name="line.142"></a>
+<span class="sourceLineNo">143</span>   * to add the step to the rollback list (or procedure stack)<a name="line.143"></a>
+<span class="sourceLineNo">144</span>   */<a name="line.144"></a>
+<span class="sourceLineNo">145</span>  protected synchronized void addRollbackStep(Procedure&lt;TEnvironment&gt; proc) {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>    if (proc.isFailed()) {<a name="line.146"></a>
+<span class="sourceLineNo">147</span>      state = State.FAILED;<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    if (subprocStack == null) {<a name="line.149"></a>
+<span class="sourceLineNo">150</span>      subprocStack = new ArrayList&lt;&gt;();<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    }<a name="line.151"></a>
+<span class="sourceLineNo">152</span>    proc.addStackIndex(subprocStack.size());<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    LOG.debug("Add procedure {} as the {}th rollback step", proc, subprocStack.size());<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    subprocStack.add(proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>  }<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span>  protected synchronized void addSubProcedure(Procedure&lt;TEnvironment&gt; proc) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    if (!proc.hasParent()) {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>      return;<a name="line.159"></a>
+<span class="sourceLineNo">160</span>    }<a name="line.160"></a>
+<span class="sourceLineNo">161</span>    if (subprocs == null) {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      subprocs = new HashSet&lt;&gt;();<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    }<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    subprocs.add(proc);<a name="line.164"></a>
+<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
+<span class="sourceLineNo">166</span><a name="line.166"></a>
+<span class="sourceLineNo">167</span>  /**<a name="line.167"></a>
+<span class="sourceLineNo">168</span>   * Called on store load by the ProcedureExecutor to load part of the stack.<a name="line.168"></a>
+<span class="sourceLineNo">169</span>   *<a name="line.169"></a>
+<span class="sourceLineNo">170</span>   * Each procedure has its own stack-positions. Which means we have to write<a name="line.170"></a>
+<span class="sourceLineNo">171</span>   * to the store only the Procedure we executed, and nothing else.<a name="line.171"></a>
+<span class="sourceLineNo">172</span>   * on load we recreate the full stack by aggregating each procedure stack-positions.<a name="line.172"></a>
+<span class="sourceLineNo">173</span>   */<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  protected synchronized void loadStack(Procedure&lt;TEnvironment&gt; proc) {<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    addSubProcedure(proc);<a name="line.175"></a>
+<span class="sourceLineNo">176</span>    int[] stackIndexes = proc.getStackIndexes();<a name="line.176"></a>
+<span class="sourceLineNo">177</span>    if (stackIndexes != null) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span>      if (subprocStack == null) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>        subprocStack = new ArrayList&lt;&gt;();<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      for (int i = 0; i &lt; stackIndexes.length; ++i) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        subprocStack.set(stackIndexes[i], proc);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>      }<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    if (proc.getState() == ProcedureState.ROLLEDBACK) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>      state = State.ROLLINGBACK;<a name="line.186"></a>
-<span class="sourceLineNo">187</span>    } else if (proc.isFailed()) {<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      state = State.FAILED;<a name="line.188"></a>
+<span class="sourceLineNo">181</span>      int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();<a name="line.181"></a>
+<span class="sourceLineNo">182</span>      if (diff &gt; 0) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>        subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>        while (diff-- &gt; 0) subprocStack.add(null);<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      }<a name="line.185"></a>
+<span class="sourceLineNo">186</span>      for (int i = 0; i &lt; stackIndexes.length; ++i) {<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        subprocStack.set(stackIndexes[i], proc);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      }<a name="line.188"></a>
 <span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>  }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span>  /**<a name="line.192"></a>
-<span class="sourceLineNo">193</span>   * Called on store load by the ProcedureExecutor to validate the procedure stack.<a name="line.193"></a>
-<span class="sourceLineNo">194</span>   */<a name="line.194"></a>
-<span class="sourceLineNo">195</span>  protected synchronized boolean isValid() {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>    if (subprocStack != null) {<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      for (Procedure&lt;TEnvironment&gt; proc : subprocStack) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        if (proc == null) {<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          return false;<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        }<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      }<a name="line.201"></a>
-<span class="sourceLineNo">202</span>    }<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    return true;<a name="line.203"></a>
-<span class="sourceLineNo">204</span>  }<a name="line.204"></a>
-<span class="sourceLineNo">205</span>}<a name="line.205"></a>
+<span class="sourceLineNo">190</span>    if (proc.getState() == ProcedureState.ROLLEDBACK) {<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      state = State.ROLLINGBACK;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    } else if (proc.isFailed()) {<a name="line.192"></a>
+<span class="sourceLineNo">193</span>      state = State.FAILED;<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
+<span class="sourceLineNo">195</span>  }<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span>  /**<a name="line.197"></a>
+<span class="sourceLineNo">198</span>   * Called on store load by the ProcedureExecutor to validate the procedure stack.<a name="line.198"></a>
+<span class="sourceLineNo">199</span>   */<a name="line.199"></a>
+<span class="sourceLineNo">200</span>  protected synchronized boolean isValid() {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    if (subprocStack != null) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span>      for (Procedure&lt;TEnvironment&gt; proc : subprocStack) {<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        if (proc == null) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>          return false;<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        }<a name="line.205"></a>
+<span class="sourceLineNo">206</span>      }<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    }<a name="line.207"></a>
+<span class="sourceLineNo">208</span>    return true;<a name="line.208"></a>
+<span class="sourceLineNo">209</span>  }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>}<a name="line.210"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
index adff387..71006ea 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/RootProcedureState.html
@@ -32,185 +32,190 @@
 <span class="sourceLineNo">024</span>import java.util.Set;<a name="line.24"></a>
 <span class="sourceLineNo">025</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.25"></a>
 <span class="sourceLineNo">026</span>import org.apache.yetus.audience.InterfaceStability;<a name="line.26"></a>
-<span class="sourceLineNo">027</span><a name="line.27"></a>
-<span class="sourceLineNo">028</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.28"></a>
+<span class="sourceLineNo">027</span>import org.slf4j.Logger;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import org.slf4j.LoggerFactory;<a name="line.28"></a>
 <span class="sourceLineNo">029</span><a name="line.29"></a>
-<span class="sourceLineNo">030</span>/**<a name="line.30"></a>
-<span class="sourceLineNo">031</span> * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".<a name="line.31"></a>
-<span class="sourceLineNo">032</span> * A "Root Procedure" is a Procedure without parent, each subprocedure will be<a name="line.32"></a>
-<span class="sourceLineNo">033</span> * added to the "Root Procedure" stack (or rollback-stack).<a name="line.33"></a>
-<span class="sourceLineNo">034</span> *<a name="line.34"></a>
-<span class="sourceLineNo">035</span> * RootProcedureState is used and managed only by the ProcedureExecutor.<a name="line.35"></a>
-<span class="sourceLineNo">036</span> *    Long rootProcId = getRootProcedureId(proc);<a name="line.36"></a>
-<span class="sourceLineNo">037</span> *    rollbackStack.get(rootProcId).acquire(proc)<a name="line.37"></a>
-<span class="sourceLineNo">038</span> *    rollbackStack.get(rootProcId).release(proc)<a name="line.38"></a>
-<span class="sourceLineNo">039</span> *    ...<a name="line.39"></a>
-<span class="sourceLineNo">040</span> */<a name="line.40"></a>
-<span class="sourceLineNo">041</span>@InterfaceAudience.Private<a name="line.41"></a>
-<span class="sourceLineNo">042</span>@InterfaceStability.Evolving<a name="line.42"></a>
-<span class="sourceLineNo">043</span>class RootProcedureState&lt;TEnvironment&gt; {<a name="line.43"></a>
-<span class="sourceLineNo">044</span><a name="line.44"></a>
-<span class="sourceLineNo">045</span>  private enum State {<a name="line.45"></a>
-<span class="sourceLineNo">046</span>    RUNNING,         // The Procedure is running or ready to run<a name="line.46"></a>
-<span class="sourceLineNo">047</span>    FAILED,          // The Procedure failed, waiting for the rollback executing<a name="line.47"></a>
-<span class="sourceLineNo">048</span>    ROLLINGBACK,     // The Procedure failed and the execution was rolledback<a name="line.48"></a>
-<span class="sourceLineNo">049</span>  }<a name="line.49"></a>
-<span class="sourceLineNo">050</span><a name="line.50"></a>
-<span class="sourceLineNo">051</span>  private Set&lt;Procedure&lt;TEnvironment&gt;&gt; subprocs = null;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>  private ArrayList&lt;Procedure&lt;TEnvironment&gt;&gt; subprocStack = null;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>  private State state = State.RUNNING;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>  private int running = 0;<a name="line.54"></a>
-<span class="sourceLineNo">055</span><a name="line.55"></a>
-<span class="sourceLineNo">056</span>  public synchronized boolean isFailed() {<a name="line.56"></a>
-<span class="sourceLineNo">057</span>    switch (state) {<a name="line.57"></a>
-<span class="sourceLineNo">058</span>      case ROLLINGBACK:<a name="line.58"></a>
-<span class="sourceLineNo">059</span>      case FAILED:<a name="line.59"></a>
-<span class="sourceLineNo">060</span>        return true;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>      default:<a name="line.61"></a>
-<span class="sourceLineNo">062</span>        break;<a name="line.62"></a>
-<span class="sourceLineNo">063</span>    }<a name="line.63"></a>
-<span class="sourceLineNo">064</span>    return false;<a name="line.64"></a>
-<span class="sourceLineNo">065</span>  }<a name="line.65"></a>
-<span class="sourceLineNo">066</span><a name="line.66"></a>
-<span class="sourceLineNo">067</span>  public synchronized boolean isRollingback() {<a name="line.67"></a>
-<span class="sourceLineNo">068</span>    return state == State.ROLLINGBACK;<a name="line.68"></a>
+<span class="sourceLineNo">030</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.30"></a>
+<span class="sourceLineNo">031</span><a name="line.31"></a>
+<span class="sourceLineNo">032</span>/**<a name="line.32"></a>
+<span class="sourceLineNo">033</span> * Internal state of the ProcedureExecutor that describes the state of a "Root Procedure".<a name="line.33"></a>
+<span class="sourceLineNo">034</span> * A "Root Procedure" is a Procedure without parent, each subprocedure will be<a name="line.34"></a>
+<span class="sourceLineNo">035</span> * added to the "Root Procedure" stack (or rollback-stack).<a name="line.35"></a>
+<span class="sourceLineNo">036</span> *<a name="line.36"></a>
+<span class="sourceLineNo">037</span> * RootProcedureState is used and managed only by the ProcedureExecutor.<a name="line.37"></a>
+<span class="sourceLineNo">038</span> *    Long rootProcId = getRootProcedureId(proc);<a name="line.38"></a>
+<span class="sourceLineNo">039</span> *    rollbackStack.get(rootProcId).acquire(proc)<a name="line.39"></a>
+<span class="sourceLineNo">040</span> *    rollbackStack.get(rootProcId).release(proc)<a name="line.40"></a>
+<span class="sourceLineNo">041</span> *    ...<a name="line.41"></a>
+<span class="sourceLineNo">042</span> */<a name="line.42"></a>
+<span class="sourceLineNo">043</span>@InterfaceAudience.Private<a name="line.43"></a>
+<span class="sourceLineNo">044</span>@InterfaceStability.Evolving<a name="line.44"></a>
+<span class="sourceLineNo">045</span>class RootProcedureState&lt;TEnvironment&gt; {<a name="line.45"></a>
+<span class="sourceLineNo">046</span><a name="line.46"></a>
+<span class="sourceLineNo">047</span>  private static final Logger LOG = LoggerFactory.getLogger(RootProcedureState.class);<a name="line.47"></a>
+<span class="sourceLineNo">048</span><a name="line.48"></a>
+<span class="sourceLineNo">049</span>  private enum State {<a name="line.49"></a>
+<span class="sourceLineNo">050</span>    RUNNING,         // The Procedure is running or ready to run<a name="line.50"></a>
+<span class="sourceLineNo">051</span>    FAILED,          // The Procedure failed, waiting for the rollback executing<a name="line.51"></a>
+<span class="sourceLineNo">052</span>    ROLLINGBACK,     // The Procedure failed and the execution was rolledback<a name="line.52"></a>
+<span class="sourceLineNo">053</span>  }<a name="line.53"></a>
+<span class="sourceLineNo">054</span><a name="line.54"></a>
+<span class="sourceLineNo">055</span>  private Set&lt;Procedure&lt;TEnvironment&gt;&gt; subprocs = null;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>  private ArrayList&lt;Procedure&lt;TEnvironment&gt;&gt; subprocStack = null;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>  private State state = State.RUNNING;<a name="line.57"></a>
+<span class="sourceLineNo">058</span>  private int running = 0;<a name="line.58"></a>
+<span class="sourceLineNo">059</span><a name="line.59"></a>
+<span class="sourceLineNo">060</span>  public synchronized boolean isFailed() {<a name="line.60"></a>
+<span class="sourceLineNo">061</span>    switch (state) {<a name="line.61"></a>
+<span class="sourceLineNo">062</span>      case ROLLINGBACK:<a name="line.62"></a>
+<span class="sourceLineNo">063</span>      case FAILED:<a name="line.63"></a>
+<span class="sourceLineNo">064</span>        return true;<a name="line.64"></a>
+<span class="sourceLineNo">065</span>      default:<a name="line.65"></a>
+<span class="sourceLineNo">066</span>        break;<a name="line.66"></a>
+<span class="sourceLineNo">067</span>    }<a name="line.67"></a>
+<span class="sourceLineNo">068</span>    return false;<a name="line.68"></a>
 <span class="sourceLineNo">069</span>  }<a name="line.69"></a>
 <span class="sourceLineNo">070</span><a name="line.70"></a>
-<span class="sourceLineNo">071</span>  /**<a name="line.71"></a>
-<span class="sourceLineNo">072</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.72"></a>
-<span class="sourceLineNo">073</span>   */<a name="line.73"></a>
-<span class="sourceLineNo">074</span>  protected synchronized boolean setRollback() {<a name="line.74"></a>
-<span class="sourceLineNo">075</span>    if (running == 0 &amp;&amp; state == State.FAILED) {<a name="line.75"></a>
-<span class="sourceLineNo">076</span>      state = State.ROLLINGBACK;<a name="line.76"></a>
-<span class="sourceLineNo">077</span>      return true;<a name="line.77"></a>
-<span class="sourceLineNo">078</span>    }<a name="line.78"></a>
-<span class="sourceLineNo">079</span>    return false;<a name="line.79"></a>
-<span class="sourceLineNo">080</span>  }<a name="line.80"></a>
-<span class="sourceLineNo">081</span><a name="line.81"></a>
-<span class="sourceLineNo">082</span>  /**<a name="line.82"></a>
-<span class="sourceLineNo">083</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.83"></a>
-<span class="sourceLineNo">084</span>   */<a name="line.84"></a>
-<span class="sourceLineNo">085</span>  protected synchronized void unsetRollback() {<a name="line.85"></a>
-<span class="sourceLineNo">086</span>    assert state == State.ROLLINGBACK;<a name="line.86"></a>
-<span class="sourceLineNo">087</span>    state = State.FAILED;<a name="line.87"></a>
-<span class="sourceLineNo">088</span>  }<a name="line.88"></a>
-<span class="sourceLineNo">089</span><a name="line.89"></a>
-<span class="sourceLineNo">090</span>  protected synchronized long[] getSubprocedureIds() {<a name="line.90"></a>
-<span class="sourceLineNo">091</span>    if (subprocs == null) {<a name="line.91"></a>
-<span class="sourceLineNo">092</span>      return null;<a name="line.92"></a>
-<span class="sourceLineNo">093</span>    }<a name="line.93"></a>
-<span class="sourceLineNo">094</span>    return subprocs.stream().mapToLong(Procedure::getProcId).toArray();<a name="line.94"></a>
-<span class="sourceLineNo">095</span>  }<a name="line.95"></a>
-<span class="sourceLineNo">096</span><a name="line.96"></a>
-<span class="sourceLineNo">097</span>  protected synchronized List&lt;Procedure&lt;TEnvironment&gt;&gt; getSubproceduresStack() {<a name="line.97"></a>
-<span class="sourceLineNo">098</span>    return subprocStack;<a name="line.98"></a>
+<span class="sourceLineNo">071</span>  public synchronized boolean isRollingback() {<a name="line.71"></a>
+<span class="sourceLineNo">072</span>    return state == State.ROLLINGBACK;<a name="line.72"></a>
+<span class="sourceLineNo">073</span>  }<a name="line.73"></a>
+<span class="sourceLineNo">074</span><a name="line.74"></a>
+<span class="sourceLineNo">075</span>  /**<a name="line.75"></a>
+<span class="sourceLineNo">076</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.76"></a>
+<span class="sourceLineNo">077</span>   */<a name="line.77"></a>
+<span class="sourceLineNo">078</span>  protected synchronized boolean setRollback() {<a name="line.78"></a>
+<span class="sourceLineNo">079</span>    if (running == 0 &amp;&amp; state == State.FAILED) {<a name="line.79"></a>
+<span class="sourceLineNo">080</span>      state = State.ROLLINGBACK;<a name="line.80"></a>
+<span class="sourceLineNo">081</span>      return true;<a name="line.81"></a>
+<span class="sourceLineNo">082</span>    }<a name="line.82"></a>
+<span class="sourceLineNo">083</span>    return false;<a name="line.83"></a>
+<span class="sourceLineNo">084</span>  }<a name="line.84"></a>
+<span class="sourceLineNo">085</span><a name="line.85"></a>
+<span class="sourceLineNo">086</span>  /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span>   * Called by the ProcedureExecutor to mark rollback execution<a name="line.87"></a>
+<span class="sourceLineNo">088</span>   */<a name="line.88"></a>
+<span class="sourceLineNo">089</span>  protected synchronized void unsetRollback() {<a name="line.89"></a>
+<span class="sourceLineNo">090</span>    assert state == State.ROLLINGBACK;<a name="line.90"></a>
+<span class="sourceLineNo">091</span>    state = State.FAILED;<a name="line.91"></a>
+<span class="sourceLineNo">092</span>  }<a name="line.92"></a>
+<span class="sourceLineNo">093</span><a name="line.93"></a>
+<span class="sourceLineNo">094</span>  protected synchronized long[] getSubprocedureIds() {<a name="line.94"></a>
+<span class="sourceLineNo">095</span>    if (subprocs == null) {<a name="line.95"></a>
+<span class="sourceLineNo">096</span>      return null;<a name="line.96"></a>
+<span class="sourceLineNo">097</span>    }<a name="line.97"></a>
+<span class="sourceLineNo">098</span>    return subprocs.stream().mapToLong(Procedure::getProcId).toArray();<a name="line.98"></a>
 <span class="sourceLineNo">099</span>  }<a name="line.99"></a>
 <span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span>  protected synchronized RemoteProcedureException getException() {<a name="line.101"></a>
-<span class="sourceLineNo">102</span>    if (subprocStack != null) {<a name="line.102"></a>
-<span class="sourceLineNo">103</span>      for (Procedure&lt;TEnvironment&gt; proc: subprocStack) {<a name="line.103"></a>
-<span class="sourceLineNo">104</span>        if (proc.hasException()) {<a name="line.104"></a>
-<span class="sourceLineNo">105</span>          return proc.getException();<a name="line.105"></a>
-<span class="sourceLineNo">106</span>        }<a name="line.106"></a>
-<span class="sourceLineNo">107</span>      }<a name="line.107"></a>
-<span class="sourceLineNo">108</span>    }<a name="line.108"></a>
-<span class="sourceLineNo">109</span>    return null;<a name="line.109"></a>
-<span class="sourceLineNo">110</span>  }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span>  /**<a name="line.112"></a>
-<span class="sourceLineNo">113</span>   * Called by the ProcedureExecutor to mark the procedure step as running.<a name="line.113"></a>
-<span class="sourceLineNo">114</span>   */<a name="line.114"></a>
-<span class="sourceLineNo">115</span>  protected synchronized boolean acquire(Procedure&lt;TEnvironment&gt; proc) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span>    if (state != State.RUNNING) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span>      return false;<a name="line.117"></a>
-<span class="sourceLineNo">118</span>    }<a name="line.118"></a>
-<span class="sourceLineNo">119</span><a name="line.119"></a>
-<span class="sourceLineNo">120</span>    running++;<a name="line.120"></a>
-<span class="sourceLineNo">121</span>    return true;<a name="line.121"></a>
-<span class="sourceLineNo">122</span>  }<a name="line.122"></a>
+<span class="sourceLineNo">101</span>  protected synchronized List&lt;Procedure&lt;TEnvironment&gt;&gt; getSubproceduresStack() {<a name="line.101"></a>
+<span class="sourceLineNo">102</span>    return subprocStack;<a name="line.102"></a>
+<span class="sourceLineNo">103</span>  }<a name="line.103"></a>
+<span class="sourceLineNo">104</span><a name="line.104"></a>
+<span class="sourceLineNo">105</span>  protected synchronized RemoteProcedureException getException() {<a name="line.105"></a>
+<span class="sourceLineNo">106</span>    if (subprocStack != null) {<a name="line.106"></a>
+<span class="sourceLineNo">107</span>      for (Procedure&lt;TEnvironment&gt; proc: subprocStack) {<a name="line.107"></a>
+<span class="sourceLineNo">108</span>        if (proc.hasException()) {<a name="line.108"></a>
+<span class="sourceLineNo">109</span>          return proc.getException();<a name="line.109"></a>
+<span class="sourceLineNo">110</span>        }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>      }<a name="line.111"></a>
+<span class="sourceLineNo">112</span>    }<a name="line.112"></a>
+<span class="sourceLineNo">113</span>    return null;<a name="line.113"></a>
+<span class="sourceLineNo">114</span>  }<a name="line.114"></a>
+<span class="sourceLineNo">115</span><a name="line.115"></a>
+<span class="sourceLineNo">116</span>  /**<a name="line.116"></a>
+<span class="sourceLineNo">117</span>   * Called by the ProcedureExecutor to mark the procedure step as running.<a name="line.117"></a>
+<span class="sourceLineNo">118</span>   */<a name="line.118"></a>
+<span class="sourceLineNo">119</span>  protected synchronized boolean acquire(Procedure&lt;TEnvironment&gt; proc) {<a name="line.119"></a>
+<span class="sourceLineNo">120</span>    if (state != State.RUNNING) {<a name="line.120"></a>
+<span class="sourceLineNo">121</span>      return false;<a name="line.121"></a>
+<span class="sourceLineNo">122</span>    }<a name="line.122"></a>
 <span class="sourceLineNo">123</span><a name="line.123"></a>
-<span class="sourceLineNo">124</span>  /**<a name="line.124"></a>
-<span class="sourceLineNo">125</span>   * Called by the ProcedureExecutor to mark the procedure step as finished.<a name="line.125"></a>
-<span class="sourceLineNo">126</span>   */<a name="line.126"></a>
-<span class="sourceLineNo">127</span>  protected synchronized void release(Procedure&lt;TEnvironment&gt; proc) {<a name="line.127"></a>
-<span class="sourceLineNo">128</span>    running--;<a name="line.128"></a>
-<span class="sourceLineNo">129</span>  }<a name="line.129"></a>
-<span class="sourceLineNo">130</span><a name="line.130"></a>
-<span class="sourceLineNo">131</span>  protected synchronized void abort() {<a name="line.131"></a>
-<span class="sourceLineNo">132</span>    if (state == State.RUNNING) {<a name="line.132"></a>
-<span class="sourceLineNo">133</span>      state = State.FAILED;<a name="line.133"></a>
-<span class="sourceLineNo">134</span>    }<a name="line.134"></a>
-<span class="sourceLineNo">135</span>  }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span>  /**<a name="line.137"></a>
-<span class="sourceLineNo">138</span>   * Called by the ProcedureExecutor after the procedure step is completed,<a name="line.138"></a>
-<span class="sourceLineNo">139</span>   * to add the step to the rollback list (or procedure stack)<a name="line.139"></a>
-<span class="sourceLineNo">140</span>   */<a name="line.140"></a>
-<span class="sourceLineNo">141</span>  protected synchronized void addRollbackStep(Procedure&lt;TEnvironment&gt; proc) {<a name="line.141"></a>
-<span class="sourceLineNo">142</span>    if (proc.isFailed()) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span>      state = State.FAILED;<a name="line.143"></a>
-<span class="sourceLineNo">144</span>    }<a name="line.144"></a>
-<span class="sourceLineNo">145</span>    if (subprocStack == null) {<a name="line.145"></a>
-<span class="sourceLineNo">146</span>      subprocStack = new ArrayList&lt;&gt;();<a name="line.146"></a>
-<span class="sourceLineNo">147</span>    }<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    proc.addStackIndex(subprocStack.size());<a name="line.148"></a>
-<span class="sourceLineNo">149</span>    subprocStack.add(proc);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>  }<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span>  protected synchronized void addSubProcedure(Procedure&lt;TEnvironment&gt; proc) {<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    if (!proc.hasParent()) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span>      return;<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    }<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    if (subprocs == null) {<a name="line.156"></a>
-<span class="sourceLineNo">157</span>      subprocs = new HashSet&lt;&gt;();<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    }<a name="line.158"></a>
-<span class="sourceLineNo">159</span>    subprocs.add(proc);<a name="line.159"></a>
-<span class="sourceLineNo">160</span>  }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span>  /**<a name="line.162"></a>
-<span class="sourceLineNo">163</span>   * Called on store load by the ProcedureExecutor to load part of the stack.<a name="line.163"></a>
-<span class="sourceLineNo">164</span>   *<a name="line.164"></a>
-<span class="sourceLineNo">165</span>   * Each procedure has its own stack-positions. Which means we have to write<a name="line.165"></a>
-<span class="sourceLineNo">166</span>   * to the store only the Procedure we executed, and nothing else.<a name="line.166"></a>
-<span class="sourceLineNo">167</span>   * on load we recreate the full stack by aggregating each procedure stack-positions.<a name="line.167"></a>
-<span class="sourceLineNo">168</span>   */<a name="line.168"></a>
-<span class="sourceLineNo">169</span>  protected synchronized void loadStack(Procedure&lt;TEnvironment&gt; proc) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    addSubProcedure(proc);<a name="line.170"></a>
-<span class="sourceLineNo">171</span>    int[] stackIndexes = proc.getStackIndexes();<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    if (stackIndexes != null) {<a name="line.172"></a>
-<span class="sourceLineNo">173</span>      if (subprocStack == null) {<a name="line.173"></a>
-<span class="sourceLineNo">174</span>        subprocStack = new ArrayList&lt;&gt;();<a name="line.174"></a>
-<span class="sourceLineNo">175</span>      }<a name="line.175"></a>
-<span class="sourceLineNo">176</span>      int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();<a name="line.176"></a>
-<span class="sourceLineNo">177</span>      if (diff &gt; 0) {<a name="line.177"></a>
-<span class="sourceLineNo">178</span>        subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);<a name="line.178"></a>
-<span class="sourceLineNo">179</span>        while (diff-- &gt; 0) subprocStack.add(null);<a name="line.179"></a>
+<span class="sourceLineNo">124</span>    running++;<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    return true;<a name="line.125"></a>
+<span class="sourceLineNo">126</span>  }<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span>  /**<a name="line.128"></a>
+<span class="sourceLineNo">129</span>   * Called by the ProcedureExecutor to mark the procedure step as finished.<a name="line.129"></a>
+<span class="sourceLineNo">130</span>   */<a name="line.130"></a>
+<span class="sourceLineNo">131</span>  protected synchronized void release(Procedure&lt;TEnvironment&gt; proc) {<a name="line.131"></a>
+<span class="sourceLineNo">132</span>    running--;<a name="line.132"></a>
+<span class="sourceLineNo">133</span>  }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span>  protected synchronized void abort() {<a name="line.135"></a>
+<span class="sourceLineNo">136</span>    if (state == State.RUNNING) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span>      state = State.FAILED;<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    }<a name="line.138"></a>
+<span class="sourceLineNo">139</span>  }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span>  /**<a name="line.141"></a>
+<span class="sourceLineNo">142</span>   * Called by the ProcedureExecutor after the procedure step is completed,<a name="line.142"></a>
+<span class="sourceLineNo">143</span>   * to add the step to the rollback list (or procedure stack)<a name="line.143"></a>
+<span class="sourceLineNo">144</span>   */<a name="line.144"></a>
+<span class="sourceLineNo">145</span>  protected synchronized void addRollbackStep(Procedure&lt;TEnvironment&gt; proc) {<a name="line.145"></a>
+<span class="sourceLineNo">146</span>    if (proc.isFailed()) {<a name="line.146"></a>
+<span class="sourceLineNo">147</span>      state = State.FAILED;<a name="line.147"></a>
+<span class="sourceLineNo">148</span>    }<a name="line.148"></a>
+<span class="sourceLineNo">149</span>    if (subprocStack == null) {<a name="line.149"></a>
+<span class="sourceLineNo">150</span>      subprocStack = new ArrayList&lt;&gt;();<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    }<a name="line.151"></a>
+<span class="sourceLineNo">152</span>    proc.addStackIndex(subprocStack.size());<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    LOG.debug("Add procedure {} as the {}th rollback step", proc, subprocStack.size());<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    subprocStack.add(proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>  }<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span>  protected synchronized void addSubProcedure(Procedure&lt;TEnvironment&gt; proc) {<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    if (!proc.hasParent()) {<a name="line.158"></a>
+<span class="sourceLineNo">159</span>      return;<a name="line.159"></a>
+<span class="sourceLineNo">160</span>    }<a name="line.160"></a>
+<span class="sourceLineNo">161</span>    if (subprocs == null) {<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      subprocs = new HashSet&lt;&gt;();<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    }<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    subprocs.add(proc);<a name="line.164"></a>
+<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
+<span class="sourceLineNo">166</span><a name="line.166"></a>
+<span class="sourceLineNo">167</span>  /**<a name="line.167"></a>
+<span class="sourceLineNo">168</span>   * Called on store load by the ProcedureExecutor to load part of the stack.<a name="line.168"></a>
+<span class="sourceLineNo">169</span>   *<a name="line.169"></a>
+<span class="sourceLineNo">170</span>   * Each procedure has its own stack-positions. Which means we have to write<a name="line.170"></a>
+<span class="sourceLineNo">171</span>   * to the store only the Procedure we executed, and nothing else.<a name="line.171"></a>
+<span class="sourceLineNo">172</span>   * on load we recreate the full stack by aggregating each procedure stack-positions.<a name="line.172"></a>
+<span class="sourceLineNo">173</span>   */<a name="line.173"></a>
+<span class="sourceLineNo">174</span>  protected synchronized void loadStack(Procedure&lt;TEnvironment&gt; proc) {<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    addSubProcedure(proc);<a name="line.175"></a>
+<span class="sourceLineNo">176</span>    int[] stackIndexes = proc.getStackIndexes();<a name="line.176"></a>
+<span class="sourceLineNo">177</span>    if (stackIndexes != null) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span>      if (subprocStack == null) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span>        subprocStack = new ArrayList&lt;&gt;();<a name="line.179"></a>
 <span class="sourceLineNo">180</span>      }<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      for (int i = 0; i &lt; stackIndexes.length; ++i) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        subprocStack.set(stackIndexes[i], proc);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>      }<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    }<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    if (proc.getState() == ProcedureState.ROLLEDBACK) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>      state = State.ROLLINGBACK;<a name="line.186"></a>
-<span class="sourceLineNo">187</span>    } else if (proc.isFailed()) {<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      state = State.FAILED;<a name="line.188"></a>
+<span class="sourceLineNo">181</span>      int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();<a name="line.181"></a>
+<span class="sourceLineNo">182</span>      if (diff &gt; 0) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>        subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);<a name="line.183"></a>
+<span class="sourceLineNo">184</span>        while (diff-- &gt; 0) subprocStack.add(null);<a name="line.184"></a>
+<span class="sourceLineNo">185</span>      }<a name="line.185"></a>
+<span class="sourceLineNo">186</span>      for (int i = 0; i &lt; stackIndexes.length; ++i) {<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        subprocStack.set(stackIndexes[i], proc);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      }<a name="line.188"></a>
 <span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>  }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span>  /**<a name="line.192"></a>
-<span class="sourceLineNo">193</span>   * Called on store load by the ProcedureExecutor to validate the procedure stack.<a name="line.193"></a>
-<span class="sourceLineNo">194</span>   */<a name="line.194"></a>
-<span class="sourceLineNo">195</span>  protected synchronized boolean isValid() {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>    if (subprocStack != null) {<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      for (Procedure&lt;TEnvironment&gt; proc : subprocStack) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        if (proc == null) {<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          return false;<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        }<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      }<a name="line.201"></a>
-<span class="sourceLineNo">202</span>    }<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    return true;<a name="line.203"></a>
-<span class="sourceLineNo">204</span>  }<a name="line.204"></a>
-<span class="sourceLineNo">205</span>}<a name="line.205"></a>
+<span class="sourceLineNo">190</span>    if (proc.getState() == ProcedureState.ROLLEDBACK) {<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      state = State.ROLLINGBACK;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    } else if (proc.isFailed()) {<a name="line.192"></a>
+<span class="sourceLineNo">193</span>      state = State.FAILED;<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    }<a name="line.194"></a>
+<span class="sourceLineNo">195</span>  }<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span>  /**<a name="line.197"></a>
+<span class="sourceLineNo">198</span>   * Called on store load by the ProcedureExecutor to validate the procedure stack.<a name="line.198"></a>
+<span class="sourceLineNo">199</span>   */<a name="line.199"></a>
+<span class="sourceLineNo">200</span>  protected synchronized boolean isValid() {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>    if (subprocStack != null) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span>      for (Procedure&lt;TEnvironment&gt; proc : subprocStack) {<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        if (proc == null) {<a name="line.203"></a>
+<span class="sourceLineNo">204</span>          return false;<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        }<a name="line.205"></a>
+<span class="sourceLineNo">206</span>      }<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    }<a name="line.207"></a>
+<span class="sourceLineNo">208</span>    return true;<a name="line.208"></a>
+<span class="sourceLineNo">209</span>  }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>}<a name="line.210"></a>
 
 
 


[44/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
index bdeb212..8e1ebb3 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>private static final class <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.221">WALProcedureTree.Iter</a>
+<pre>private static final class <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.224">WALProcedureTree.Iter</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
 implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a></pre>
 </li>
@@ -253,7 +253,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>procs</h4>
-<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.223">procs</a></pre>
+<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.226">procs</a></pre>
 </li>
 </ul>
 <a name="iter">
@@ -262,7 +262,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>iter</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.225">iter</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt; <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.228">iter</a></pre>
 </li>
 </ul>
 <a name="current">
@@ -271,7 +271,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockListLast">
 <li class="blockList">
 <h4>current</h4>
-<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.227">current</a></pre>
+<pre>private&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a> <a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.230">current</a></pre>
 </li>
 </ul>
 </li>
@@ -288,7 +288,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockListLast">
 <li class="blockList">
 <h4>Iter</h4>
-<pre>public&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.229">Iter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt;&nbsp;procs)</pre>
+<pre>public&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.232">Iter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.ProtoAndProc</a>&gt;&nbsp;procs)</pre>
 </li>
 </ul>
 </li>
@@ -305,7 +305,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>reset</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.235">reset</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.238">reset</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#reset--">ProcedureStore.ProcedureIterator</a></code></span></div>
 <div class="block">Reset the Iterator by seeking to the beginning of the list.</div>
 <dl>
@@ -320,7 +320,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>hasNext</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.245">hasNext</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.248">hasNext</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#hasNext--">ProcedureStore.ProcedureIterator</a></code></span></div>
 <div class="block">Returns true if the iterator has more elements.
  (In other words, returns true if next() would return a Procedure
@@ -339,7 +339,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>checkNext</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.249">checkNext</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.252">checkNext</a>()</pre>
 </li>
 </ul>
 <a name="isNextFinished--">
@@ -348,7 +348,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>isNextFinished</h4>
-<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.256">isNextFinished</a>()</pre>
+<pre>public&nbsp;boolean&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.259">isNextFinished</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#isNextFinished--">ProcedureStore.ProcedureIterator</a></code></span></div>
 <div class="block">Calling this method does not need to convert the protobuf message to the Procedure class, so
  if it returns true we can call <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#skipNext--"><code>ProcedureStore.ProcedureIterator.skipNext()</code></a> to skip the procedure without
@@ -367,7 +367,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>moveToNext</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.261">moveToNext</a>()</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.264">moveToNext</a>()</pre>
 </li>
 </ul>
 <a name="skipNext--">
@@ -376,7 +376,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockList">
 <li class="blockList">
 <h4>skipNext</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.270">skipNext</a>()</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.273">skipNext</a>()</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#skipNext--">ProcedureStore.ProcedureIterator</a></code></span></div>
 <div class="block">Skip the next procedure
  <p/>
@@ -394,7 +394,7 @@ implements <a href="../../../../../../../org/apache/hadoop/hbase/procedure2/stor
 <ul class="blockListLast">
 <li class="blockList">
 <h4>next</h4>
-<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.276">next</a>()
+<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>&lt;?&gt;&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html#line.279">next</a>()
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html#next--">ProcedureStore.ProcedureIterator</a></code></span></div>
 <div class="block">Returns the next procedure in the iteration.</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
index 16ed09d..2ce8cb0 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
@@ -367,7 +367,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>addAllToCorruptedAndRemoveFromProcMap</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.158">addAllToCorruptedAndRemoveFromProcMap</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;entry,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.161">addAllToCorruptedAndRemoveFromProcMap</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;entry,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&gt;&nbsp;remainingProcMap)</pre>
 </li>
 </ul>
@@ -377,7 +377,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>addAllToValidAndRemoveFromProcMap</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.167">addAllToValidAndRemoveFromProcMap</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;entry,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.170">addAllToValidAndRemoveFromProcMap</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;entry,
                                                <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&gt;&nbsp;remainingProcMap)</pre>
 </li>
 </ul>
@@ -387,7 +387,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>checkReady</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.179">checkReady</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;rootEntry,
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.182">checkReady</a>(<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&nbsp;rootEntry,
                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&gt;&nbsp;remainingProcMap)</pre>
 </li>
 </ul>
@@ -397,7 +397,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>checkOrphan</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.214">checkOrphan</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&gt;&nbsp;procMap)</pre>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.217">checkOrphan</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Entry.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree.Entry</a>&gt;&nbsp;procMap)</pre>
 </li>
 </ul>
 <a name="getValidProcs--">
@@ -406,7 +406,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getValidProcs</h4>
-<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.284">getValidProcs</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.287">getValidProcs</a>()</pre>
 </li>
 </ul>
 <a name="getCorruptedProcs--">
@@ -415,7 +415,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getCorruptedProcs</h4>
-<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.288">getCorruptedProcs</a>()</pre>
+<pre>public&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.291">getCorruptedProcs</a>()</pre>
 </li>
 </ul>
 <a name="build-java.util.Collection-">
@@ -424,7 +424,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>build</h4>
-<pre>public static&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.292">build</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedure&gt;&nbsp;procedures)</pre>
+<pre>public static&nbsp;<a href="../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">WALProcedureTree</a>&nbsp;<a href="../../../../../../../src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html#line.295">build</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedure&gt;&nbsp;procedures)</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFile.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFile.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFile.html
index cb7f6d8..fcdf2b3 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFile.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFile.html
@@ -187,7 +187,10 @@
 <td class="colFirst"><code>static void</code></td>
 <td class="colLast"><span class="typeNameLabel">ProcedureWALFormat.</span><code><span class="memberNameLink"><a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#load-java.util.Iterator-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.Loader-">load</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
     <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-    <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>&nbsp;</td>
+    <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.</div>
+</td>
 </tr>
 </tbody>
 </table>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFormat.Loader.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFormat.Loader.html b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFormat.Loader.html
index 97ac41c..394e2c0 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFormat.Loader.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/store/wal/class-use/ProcedureWALFormat.Loader.html
@@ -119,7 +119,10 @@
 <td class="colFirst"><code>static void</code></td>
 <td class="colLast"><span class="typeNameLabel">ProcedureWALFormat.</span><code><span class="memberNameLink"><a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.html#load-java.util.Iterator-org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker-org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.Loader-">load</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html?is-external=true" title="class or interface in java.util">Iterator</a>&lt;<a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFile.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFile</a>&gt;&nbsp;logs,
     <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStoreTracker.html" title="class in org.apache.hadoop.hbase.procedure2.store">ProcedureStoreTracker</a>&nbsp;tracker,
-    <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>&nbsp;</td>
+    <a href="../../../../../../../../org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.Loader.html" title="interface in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormat.Loader</a>&nbsp;loader)</code>
+<div class="block">Load all the procedures in these ProcedureWALFiles, and rebuild the given <code>tracker</code> if
+ needed, i.e, the <code>tracker</code> is a partial one.</div>
+</td>
 </tr>
 </tbody>
 </table>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
index d4d6c86..f7ea06e 100644
--- a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
@@ -229,13 +229,13 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottlingException.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/OperationQuota.OperationType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">OperationQuota.OperationType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaScope.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaScope</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottleType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottleType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottleType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottleType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaScope.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaScope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/OperationQuota.OperationType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">OperationQuota.OperationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottlingException.Type</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/RpcThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">RpcThrottlingException.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html b/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
index de67223..64b462b 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.288">SecureBulkLoadManager.SecureBulkLoadListener</a>
+<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.331">SecureBulkLoadManager.SecureBulkLoadListener</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
 implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html" title="interface in org.apache.hadoop.hbase.regionserver">HRegion.BulkLoadListener</a></pre>
 </li>
@@ -252,7 +252,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>fs</h4>
-<pre>private final&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.290">fs</a></pre>
+<pre>private final&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.333">fs</a></pre>
 </li>
 </ul>
 <a name="stagingDir">
@@ -261,7 +261,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>stagingDir</h4>
-<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.291">stagingDir</a></pre>
+<pre>private final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.334">stagingDir</a></pre>
 </li>
 </ul>
 <a name="conf">
@@ -270,7 +270,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>conf</h4>
-<pre>private final&nbsp;org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.292">conf</a></pre>
+<pre>private final&nbsp;org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.335">conf</a></pre>
 </li>
 </ul>
 <a name="srcFs">
@@ -279,7 +279,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>srcFs</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.294">srcFs</a></pre>
+<pre>private&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.337">srcFs</a></pre>
 </li>
 </ul>
 <a name="origPermissions">
@@ -288,7 +288,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockListLast">
 <li class="blockList">
 <h4>origPermissions</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,org.apache.hadoop.fs.permission.FsPermission&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.295">origPermissions</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,org.apache.hadoop.fs.permission.FsPermission&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.338">origPermissions</a></pre>
 </li>
 </ul>
 </li>
@@ -305,7 +305,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockListLast">
 <li class="blockList">
 <h4>SecureBulkLoadListener</h4>
-<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.297">SecureBulkLoadListener</a>(org.apache.hadoop.fs.FileSystem&nbsp;fs,
+<pre>public&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.340">SecureBulkLoadListener</a>(org.apache.hadoop.fs.FileSystem&nbsp;fs,
                               <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;stagingDir,
                               org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
@@ -324,7 +324,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>prepareBulkLoad</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.305">prepareBulkLoad</a>(byte[]&nbsp;family,
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.348">prepareBulkLoad</a>(byte[]&nbsp;family,
                               <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;srcPath,
                               boolean&nbsp;copyFile)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -349,7 +349,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>doneBulkLoad</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.346">doneBulkLoad</a>(byte[]&nbsp;family,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.389">doneBulkLoad</a>(byte[]&nbsp;family,
                          <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;srcPath)
                   throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html#doneBulkLoad-byte:A-java.lang.String-">HRegion.BulkLoadListener</a></code></span></div>
@@ -371,7 +371,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>closeSrcFs</h4>
-<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.351">closeSrcFs</a>()
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.394">closeSrcFs</a>()
                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -385,7 +385,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockList">
 <li class="blockList">
 <h4>failedBulkLoad</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.359">failedBulkLoad</a>(byte[]&nbsp;family,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.402">failedBulkLoad</a>(byte[]&nbsp;family,
                            <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;srcPath)
                     throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.BulkLoadListener.html#failedBulkLoad-byte:A-java.lang.String-">HRegion.BulkLoadListener</a></code></span></div>
@@ -407,7 +407,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.
 <ul class="blockListLast">
 <li class="blockList">
 <h4>isFile</h4>
-<pre>private&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.401">isFile</a>(org.apache.hadoop.fs.Path&nbsp;p)
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.SecureBulkLoadListener.html#line.444">isFile</a>(org.apache.hadoop.fs.Path&nbsp;p)
                 throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <div class="block">Check if the path is referencing a file.
  This is mainly needed to avoid symlinks.</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html b/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
index 858be19..0874e43 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html
@@ -18,7 +18,7 @@
     catch(err) {
     }
 //-->
-var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10};
 var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
 var altColor = "altColor";
 var rowColor = "rowColor";
@@ -110,7 +110,7 @@ var activeTableTab = "activeTableTab";
 <hr>
 <br>
 <pre>@InterfaceAudience.Private
-public class <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.88">SecureBulkLoadManager</a>
+public class <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.92">SecureBulkLoadManager</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Bulk loads in secure mode.
 
@@ -189,29 +189,37 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#fs">fs</a></span></code>&nbsp;</td>
 </tr>
 <tr class="altColor">
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#fsCreatedListener">fsCreatedListener</a></span></code>&nbsp;</td>
+</tr>
+<tr class="rowColor">
 <td class="colFirst"><code>private static org.slf4j.Logger</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#LOG">LOG</a></span></code>&nbsp;</td>
 </tr>
-<tr class="rowColor">
+<tr class="altColor">
 <td class="colFirst"><code>private static org.apache.hadoop.fs.permission.FsPermission</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#PERM_ALL_ACCESS">PERM_ALL_ACCESS</a></span></code>&nbsp;</td>
 </tr>
-<tr class="altColor">
+<tr class="rowColor">
 <td class="colFirst"><code>private static org.apache.hadoop.fs.permission.FsPermission</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#PERM_HIDDEN">PERM_HIDDEN</a></span></code>&nbsp;</td>
 </tr>
-<tr class="rowColor">
+<tr class="altColor">
 <td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/security/SecureRandom.html?is-external=true" title="class or interface in java.security">SecureRandom</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#random">random</a></span></code>&nbsp;</td>
 </tr>
-<tr class="altColor">
+<tr class="rowColor">
 <td class="colFirst"><code>private static int</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#RANDOM_RADIX">RANDOM_RADIX</a></span></code>&nbsp;</td>
 </tr>
-<tr class="rowColor">
+<tr class="altColor">
 <td class="colFirst"><code>private static int</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#RANDOM_WIDTH">RANDOM_WIDTH</a></span></code>&nbsp;</td>
 </tr>
+<tr class="rowColor">
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a>&lt;org.apache.hadoop.security.UserGroupInformation,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>&gt;</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#ugiReferenceCounter">ugiReferenceCounter</a></span></code>&nbsp;</td>
+</tr>
 <tr class="altColor">
 <td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/security/UserProvider.html" title="class in org.apache.hadoop.hbase.security">UserProvider</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#userProvider">userProvider</a></span></code>&nbsp;</td>
@@ -271,24 +279,40 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
                 <a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)</code>&nbsp;</td>
 </tr>
 <tr id="i3" class="rowColor">
+<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#decrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">decrementUgiReference</a></span>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</code>&nbsp;</td>
+</tr>
+<tr id="i4" class="altColor">
 <td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#getActiveUser--">getActiveUser</a></span>()</code>&nbsp;</td>
 </tr>
-<tr id="i4" class="altColor">
+<tr id="i5" class="rowColor">
+<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#incrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">incrementUgiReference</a></span>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</code>&nbsp;</td>
+</tr>
+<tr id="i6" class="altColor">
+<td class="colFirst"><code>private boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#isUserReferenced-org.apache.hadoop.security.UserGroupInformation-">isUserReferenced</a></span>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</code>&nbsp;</td>
+</tr>
+<tr id="i7" class="rowColor">
 <td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#prepareBulkLoad-org.apache.hadoop.hbase.regionserver.HRegion-org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest-">prepareBulkLoad</a></span>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
                org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest&nbsp;request)</code>&nbsp;</td>
 </tr>
-<tr id="i5" class="rowColor">
+<tr id="i8" class="altColor">
 <td class="colFirst"><code><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;org.apache.hadoop.fs.Path&gt;&gt;</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#secureBulkLoadHFiles-org.apache.hadoop.hbase.regionserver.HRegion-org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest-">secureBulkLoadHFiles</a></span>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
                     org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest&nbsp;request)</code>&nbsp;</td>
 </tr>
-<tr id="i6" class="altColor">
+<tr id="i9" class="rowColor">
+<td class="colFirst"><code>(package private) void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#setFsCreatedListener-java.util.function.Consumer-">setFsCreatedListener</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;&nbsp;fsCreatedListener)</code>&nbsp;</td>
+</tr>
+<tr id="i10" class="altColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#start--">start</a></span>()</code>&nbsp;</td>
 </tr>
-<tr id="i7" class="rowColor">
+<tr id="i11" class="rowColor">
 <td class="colFirst"><code>void</code></td>
 <td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#stop--">stop</a></span>()</code>&nbsp;</td>
 </tr>
@@ -320,7 +344,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>VERSION</h4>
-<pre>public static final&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.90">VERSION</a></pre>
+<pre>public static final&nbsp;long <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.94">VERSION</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.VERSION">Constant Field Values</a></dd>
@@ -333,7 +357,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>RANDOM_WIDTH</h4>
-<pre>private static final&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.93">RANDOM_WIDTH</a></pre>
+<pre>private static final&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.97">RANDOM_WIDTH</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.RANDOM_WIDTH">Constant Field Values</a></dd>
@@ -346,7 +370,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>RANDOM_RADIX</h4>
-<pre>private static final&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.94">RANDOM_RADIX</a></pre>
+<pre>private static final&nbsp;int <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.98">RANDOM_RADIX</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.regionserver.SecureBulkLoadManager.RANDOM_RADIX">Constant Field Values</a></dd>
@@ -359,7 +383,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>LOG</h4>
-<pre>private static final&nbsp;org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.96">LOG</a></pre>
+<pre>private static final&nbsp;org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.100">LOG</a></pre>
 </li>
 </ul>
 <a name="PERM_ALL_ACCESS">
@@ -368,7 +392,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>PERM_ALL_ACCESS</h4>
-<pre>private static final&nbsp;org.apache.hadoop.fs.permission.FsPermission <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.98">PERM_ALL_ACCESS</a></pre>
+<pre>private static final&nbsp;org.apache.hadoop.fs.permission.FsPermission <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.102">PERM_ALL_ACCESS</a></pre>
 </li>
 </ul>
 <a name="PERM_HIDDEN">
@@ -377,7 +401,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>PERM_HIDDEN</h4>
-<pre>private static final&nbsp;org.apache.hadoop.fs.permission.FsPermission <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.99">PERM_HIDDEN</a></pre>
+<pre>private static final&nbsp;org.apache.hadoop.fs.permission.FsPermission <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.103">PERM_HIDDEN</a></pre>
 </li>
 </ul>
 <a name="random">
@@ -386,7 +410,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>random</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/security/SecureRandom.html?is-external=true" title="class or interface in java.security">SecureRandom</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.100">random</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/security/SecureRandom.html?is-external=true" title="class or interface in java.security">SecureRandom</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.104">random</a></pre>
 </li>
 </ul>
 <a name="fs">
@@ -395,7 +419,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>fs</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.101">fs</a></pre>
+<pre>private&nbsp;org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.105">fs</a></pre>
 </li>
 </ul>
 <a name="conf">
@@ -404,7 +428,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>conf</h4>
-<pre>private&nbsp;org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.102">conf</a></pre>
+<pre>private&nbsp;org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.106">conf</a></pre>
 </li>
 </ul>
 <a name="baseStagingDir">
@@ -413,7 +437,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>baseStagingDir</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.106">baseStagingDir</a></pre>
+<pre>private&nbsp;org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.110">baseStagingDir</a></pre>
 </li>
 </ul>
 <a name="userProvider">
@@ -422,16 +446,34 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>userProvider</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/security/UserProvider.html" title="class in org.apache.hadoop.hbase.security">UserProvider</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.108">userProvider</a></pre>
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/security/UserProvider.html" title="class in org.apache.hadoop.hbase.security">UserProvider</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.112">userProvider</a></pre>
+</li>
+</ul>
+<a name="ugiReferenceCounter">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>ugiReferenceCounter</h4>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a>&lt;org.apache.hadoop.security.UserGroupInformation,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.113">ugiReferenceCounter</a></pre>
 </li>
 </ul>
 <a name="conn">
 <!--   -->
 </a>
-<ul class="blockListLast">
+<ul class="blockList">
 <li class="blockList">
 <h4>conn</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/client/Connection.html" title="interface in org.apache.hadoop.hbase.client">Connection</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.109">conn</a></pre>
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/client/Connection.html" title="interface in org.apache.hadoop.hbase.client">Connection</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.114">conn</a></pre>
+</li>
+</ul>
+<a name="fsCreatedListener">
+<!--   -->
+</a>
+<ul class="blockListLast">
+<li class="blockList">
+<h4>fsCreatedListener</h4>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt; <a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.176">fsCreatedListener</a></pre>
 </li>
 </ul>
 </li>
@@ -448,7 +490,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>SecureBulkLoadManager</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.111">SecureBulkLoadManager</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.116">SecureBulkLoadManager</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                       <a href="../../../../../org/apache/hadoop/hbase/client/Connection.html" title="interface in org.apache.hadoop.hbase.client">Connection</a>&nbsp;conn)</pre>
 </li>
 </ul>
@@ -466,7 +508,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>start</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.116">start</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.121">start</a>()
            throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -480,7 +522,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>stop</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.131">stop</a>()
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.137">stop</a>()
           throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>
@@ -494,7 +536,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>prepareBulkLoad</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.134">prepareBulkLoad</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.140">prepareBulkLoad</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
                               org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest&nbsp;request)
                        throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -509,7 +551,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>cleanupBulkLoad</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.146">cleanupBulkLoad</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
+<pre>public&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.152">cleanupBulkLoad</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
                             org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest&nbsp;request)
                      throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -518,13 +560,49 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 </dl>
 </li>
 </ul>
+<a name="setFsCreatedListener-java.util.function.Consumer-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>setFsCreatedListener</h4>
+<pre>void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.179">setFsCreatedListener</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;&nbsp;fsCreatedListener)</pre>
+</li>
+</ul>
+<a name="incrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>incrementUgiReference</h4>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.184">incrementUgiReference</a>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</pre>
+</li>
+</ul>
+<a name="decrementUgiReference-org.apache.hadoop.security.UserGroupInformation-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>decrementUgiReference</h4>
+<pre>private&nbsp;void&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.193">decrementUgiReference</a>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</pre>
+</li>
+</ul>
+<a name="isUserReferenced-org.apache.hadoop.security.UserGroupInformation-">
+<!--   -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>isUserReferenced</h4>
+<pre>private&nbsp;boolean&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.203">isUserReferenced</a>(org.apache.hadoop.security.UserGroupInformation&nbsp;ugi)</pre>
+</li>
+</ul>
 <a name="secureBulkLoadHFiles-org.apache.hadoop.hbase.regionserver.HRegion-org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest-">
 <!--   -->
 </a>
 <ul class="blockList">
 <li class="blockList">
 <h4>secureBulkLoadHFiles</h4>
-<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;org.apache.hadoop.fs.Path&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.170">secureBulkLoadHFiles</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
+<pre>public&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;org.apache.hadoop.fs.Path&gt;&gt;&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.208">secureBulkLoadHFiles</a>(<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&nbsp;region,
                                                                         org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest&nbsp;request)
                                                                  throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
@@ -539,7 +617,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>createStagingDir</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.258">createStagingDir</a>(org.apache.hadoop.fs.Path&nbsp;baseDir,
+<pre>private&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.301">createStagingDir</a>(org.apache.hadoop.fs.Path&nbsp;baseDir,
                                                    <a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a>&nbsp;user,
                                                    <a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a>&nbsp;tableName)
                                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -555,7 +633,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>createStagingDir</h4>
-<pre>private&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.267">createStagingDir</a>(org.apache.hadoop.fs.Path&nbsp;baseDir,
+<pre>private&nbsp;org.apache.hadoop.fs.Path&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.310">createStagingDir</a>(org.apache.hadoop.fs.Path&nbsp;baseDir,
                                                    <a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a>&nbsp;user,
                                                    <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;randomDir)
                                             throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -571,7 +649,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getActiveUser</h4>
-<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.276">getActiveUser</a>()
+<pre>private&nbsp;<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a>&nbsp;<a href="../../../../../src-html/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#line.319">getActiveUser</a>()
                     throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
 <dl>
 <dt><span class="throwsLabel">Throws:</span></dt>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/HRegion.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/HRegion.html b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/HRegion.html
index a2d4870..3e892ef 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/HRegion.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/HRegion.html
@@ -319,6 +319,10 @@
 </tr>
 <tbody>
 <tr class="altColor">
+<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;</code></td>
+<td class="colLast"><span class="typeNameLabel">SecureBulkLoadManager.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#fsCreatedListener">fsCreatedListener</a></span></code>&nbsp;</td>
+</tr>
+<tr class="rowColor">
 <td class="colFirst"><code>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;</code></td>
 <td class="colLast"><span class="typeNameLabel">HRegionServer.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/regionserver/HRegionServer.html#onlineRegions">onlineRegions</a></span></code>
 <div class="block">Map of regions currently being served by this region server.</div>
@@ -1052,6 +1056,10 @@
                         <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;&nbsp;excludedRegions,
                         boolean&nbsp;checkStoreFileCount)</code>&nbsp;</td>
 </tr>
+<tr class="rowColor">
+<td class="colFirst"><code>(package private) void</code></td>
+<td class="colLast"><span class="typeNameLabel">SecureBulkLoadManager.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.html#setFsCreatedListener-java.util.function.Consumer-">setFsCreatedListener</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html?is-external=true" title="class or interface in java.util.function">Consumer</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/regionserver/HRegion.html" title="class in org.apache.hadoop.hbase.regionserver">HRegion</a>&gt;&nbsp;fsCreatedListener)</code>&nbsp;</td>
+</tr>
 </tbody>
 </table>
 <table class="useSummary" border="0" cellpadding="3" cellspacing="0" summary="Use table, listing constructors, and an explanation">

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
index 4a0fe38..202cc24 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
@@ -711,18 +711,18 @@
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.NextState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/SplitLogWorker.TaskExecutor.Status.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">SplitLogWorker.TaskExecutor.Status</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/BloomType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">BloomType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MetricsRegionServerSourceFactoryImpl.FactoryStorage</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.StepDirection.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">DefaultHeapMemoryTuner.StepDirection</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">HRegion.FlushResult.Result</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/FlushType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">FlushType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/Region.Operation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">Region.Operation</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChunkCreator.ChunkType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ChunkCreator.ChunkType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/FlushType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">FlushType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/BloomType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">BloomType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/SplitLogWorker.TaskExecutor.Status.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">SplitLogWorker.TaskExecutor.Status</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">HRegion.FlushResult.Result</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">CompactingMemStore.IndexType</span></a></li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
index 6851ee1..b377318 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
@@ -130,9 +130,9 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.DeleteResult.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">DeleteTracker.DeleteResult</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.MatchCode.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">ScanQueryMatcher.MatchCode</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/StripeCompactionScanQueryMatcher.DropDeletesInOutput.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">StripeCompactionScanQueryMatcher.DropDeletesInOutput</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.MatchCode.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">ScanQueryMatcher.MatchCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.DeleteResult.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">DeleteTracker.DeleteResult</span></a></li>
 </ul>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/replication/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/replication/package-tree.html b/devapidocs/org/apache/hadoop/hbase/replication/package-tree.html
index 3c7146a..f8e4b11 100644
--- a/devapidocs/org/apache/hadoop/hbase/replication/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/replication/package-tree.html
@@ -160,8 +160,8 @@
 <ul>
 <li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a>&lt;E&gt; (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a>&lt;T&gt;, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
 <ul>
-<li type="circle">org.apache.hadoop.hbase.replication.<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationPeer.PeerState.html" title="enum in org.apache.hadoop.hbase.replication"><span class="typeNameLink">ReplicationPeer.PeerState</span></a></li>
 <li type="circle">org.apache.hadoop.hbase.replication.<a href="../../../../../org/apache/hadoop/hbase/replication/SyncReplicationState.html" title="enum in org.apache.hadoop.hbase.replication"><span class="typeNameLink">SyncReplicationState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.replication.<a href="../../../../../org/apache/hadoop/hbase/replication/ReplicationPeer.PeerState.html" title="enum in org.apache.hadoop.hbase.replication"><span class="typeNameLink">ReplicationPeer.PeerState</span></a></li>
 </ul>
 </li>
 </ul>


[27/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">835</span>            get

<TRUNCATED>

[48/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
index a846852..5e2481d 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html
@@ -128,7 +128,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1479">StochasticLoadBalancer.CPRequestCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1487">StochasticLoadBalancer.CPRequestCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a></pre>
 <div class="block">Compute the cost of total number of coprocessor requests  The more unbalanced the higher the
  computed cost will be.  This uses a rolling average of regionload.</div>
@@ -250,7 +250,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>CP_REQUEST_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1481">CP_REQUEST_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1489">CP_REQUEST_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.CPRequestCostFunction.CP_REQUEST_COST_KEY">Constant Field Values</a></dd>
@@ -263,7 +263,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_CP_REQUEST_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1483">DEFAULT_CP_REQUEST_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1491">DEFAULT_CP_REQUEST_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.CPRequestCostFunction.DEFAULT_CP_REQUEST_COST">Constant Field Values</a></dd>
@@ -284,7 +284,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>CPRequestCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1485">CPRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1493">CPRequestCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -301,7 +301,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1491">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CPRequestCostFunction.html#line.1499">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">getCostFromRl</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
index ea8fb44..d0cd75c 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.605">StochasticLoadBalancer.CandidateGenerator</a>
+<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.613">StochasticLoadBalancer.CandidateGenerator</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Generates a candidate action to be applied to the cluster for cost function search</div>
 </li>
@@ -232,7 +232,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>CandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.605">CandidateGenerator</a>()</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.613">CandidateGenerator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -249,7 +249,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>generate</h4>
-<pre>abstract&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.606">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>abstract&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.614">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 </li>
 </ul>
 <a name="pickRandomRegion-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-int-double-">
@@ -258,7 +258,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickRandomRegion</h4>
-<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.620">pickRandomRegion</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.628">pickRandomRegion</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                int&nbsp;server,
                                double&nbsp;chanceOfNoSwap)</pre>
 <div class="block">From a list of regions pick a random one. Null can be returned which
@@ -282,7 +282,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickRandomServer</h4>
-<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.630">pickRandomServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.638">pickRandomServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 </li>
 </ul>
 <a name="pickRandomRack-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">
@@ -291,7 +291,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickRandomRack</h4>
-<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.638">pickRandomRack</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.646">pickRandomRack</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 </li>
 </ul>
 <a name="pickOtherRandomServer-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-int-">
@@ -300,7 +300,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickOtherRandomServer</h4>
-<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.646">pickOtherRandomServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.654">pickOtherRandomServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                     int&nbsp;serverIndex)</pre>
 </li>
 </ul>
@@ -310,7 +310,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickOtherRandomRack</h4>
-<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.658">pickOtherRandomRack</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.666">pickOtherRandomRack</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                   int&nbsp;rackIndex)</pre>
 </li>
 </ul>
@@ -320,7 +320,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>pickRandomRegions</h4>
-<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.670">pickRandomRegions</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.678">pickRandomRegions</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                                             int&nbsp;thisServer,
                                                             int&nbsp;otherServer)</pre>
 </li>
@@ -331,7 +331,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getAction</h4>
-<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.691">getAction</a>(int&nbsp;fromServer,
+<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.699">getAction</a>(int&nbsp;fromServer,
                                                     int&nbsp;fromRegion,
                                                     int&nbsp;toServer,
                                                     int&nbsp;toRegion)</pre>
@@ -343,7 +343,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getRandomIterationOrder</h4>
-<pre>protected&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.711">getRandomIterationOrder</a>(int&nbsp;length)</pre>
+<pre>protected&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Integer.html?is-external=true" title="class or interface in java.lang">Integer</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#line.719">getRandomIterationOrder</a>(int&nbsp;length)</pre>
 <div class="block">Returns a random iteration order of indexes of an array with size length</div>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
index cb4eb9b..e07a6ff 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1428">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a>
+<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1436">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></pre>
 <div class="block">Class to be used for the subset of RegionLoad costs that should be treated as rates.
  We do not compare about the actual rate in requests per second but rather the rate relative
@@ -228,7 +228,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>CostFromRegionLoadAsRateFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html#line.1430">CostFromRegionLoadAsRateFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html#line.1438">CostFromRegionLoadAsRateFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -245,7 +245,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getRegionLoadCost</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html#line.1435">getRegionLoadCost</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&nbsp;regionLoadList)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html#line.1443">getRegionLoadCost</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&nbsp;regionLoadList)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getRegionLoadCost-java.util.Collection-">getRegionLoadCost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
index b6e5205..6ee1377 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1363">StochasticLoadBalancer.CostFromRegionLoadFunction</a>
+<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1371">StochasticLoadBalancer.CostFromRegionLoadFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Base class the allows writing costs functions from rolling average of some
  number from RegionLoad.</div>
@@ -250,7 +250,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>clusterStatus</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1365">clusterStatus</a></pre>
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1373">clusterStatus</a></pre>
 </li>
 </ul>
 <a name="loads">
@@ -259,7 +259,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>loads</h4>
-<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&gt; <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1366">loads</a></pre>
+<pre>private&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&gt; <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1374">loads</a></pre>
 </li>
 </ul>
 <a name="stats">
@@ -268,7 +268,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>stats</h4>
-<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1367">stats</a></pre>
+<pre>private&nbsp;double[] <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1375">stats</a></pre>
 </li>
 </ul>
 </li>
@@ -285,7 +285,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>CostFromRegionLoadFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1368">CostFromRegionLoadFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1376">CostFromRegionLoadFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -302,7 +302,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>setClusterMetrics</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1372">setClusterMetrics</a>(<a href="../../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;status)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1380">setClusterMetrics</a>(<a href="../../../../../../org/apache/hadoop/hbase/ClusterMetrics.html" title="interface in org.apache.hadoop.hbase">ClusterMetrics</a>&nbsp;status)</pre>
 </li>
 </ul>
 <a name="setLoads-java.util.Map-">
@@ -311,7 +311,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>setLoads</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1376">setLoads</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&gt;&nbsp;l)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1384">setLoads</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a>&lt;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&gt;&nbsp;l)</pre>
 </li>
 </ul>
 <a name="cost--">
@@ -320,7 +320,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1381">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1389">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>
@@ -333,7 +333,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>getRegionLoadCost</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1412">getRegionLoadCost</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&nbsp;regionLoadList)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1420">getRegionLoadCost</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a>&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&gt;&nbsp;regionLoadList)</pre>
 </li>
 </ul>
 <a name="getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">
@@ -342,7 +342,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected abstract&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1420">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected abstract&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#line.1428">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
index 3722d00..8309d36 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.966">StochasticLoadBalancer.CostFunction</a>
+<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.974">StochasticLoadBalancer.CostFunction</a>
 extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
 <div class="block">Base class of StochasticLoadBalancer's Cost Functions.</div>
 </li>
@@ -260,7 +260,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>multiplier</h4>
-<pre>private&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.968">multiplier</a></pre>
+<pre>private&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.976">multiplier</a></pre>
 </li>
 </ul>
 <a name="cluster">
@@ -269,7 +269,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cluster</h4>
-<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.970">cluster</a></pre>
+<pre>protected&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.978">cluster</a></pre>
 </li>
 </ul>
 </li>
@@ -286,7 +286,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>CostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.972">CostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;c)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.980">CostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;c)</pre>
 </li>
 </ul>
 </li>
@@ -303,7 +303,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>isNeeded</h4>
-<pre>boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.975">isNeeded</a>()</pre>
+<pre>boolean&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.983">isNeeded</a>()</pre>
 </li>
 </ul>
 <a name="getMultiplier--">
@@ -312,7 +312,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getMultiplier</h4>
-<pre>float&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.978">getMultiplier</a>()</pre>
+<pre>float&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.986">getMultiplier</a>()</pre>
 </li>
 </ul>
 <a name="setMultiplier-float-">
@@ -321,7 +321,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>setMultiplier</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.982">setMultiplier</a>(float&nbsp;m)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.990">setMultiplier</a>(float&nbsp;m)</pre>
 </li>
 </ul>
 <a name="init-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">
@@ -330,7 +330,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>init</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.989">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.997">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <div class="block">Called once per LB invocation to give the cost function
  to initialize it's state, and perform any costly calculation.</div>
 </li>
@@ -341,7 +341,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>postAction</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.997">postAction</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;action)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1005">postAction</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;action)</pre>
 <div class="block">Called once per cluster Action to give the cost function
  an opportunity to update it's state. postAction() is always
  called at least once before cost() is called with the cluster
@@ -354,7 +354,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>regionMoved</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1018">regionMoved</a>(int&nbsp;region,
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1026">regionMoved</a>(int&nbsp;region,
                            int&nbsp;oldServer,
                            int&nbsp;newServer)</pre>
 </li>
@@ -365,7 +365,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>cost</h4>
-<pre>abstract&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1021">cost</a>()</pre>
+<pre>abstract&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1029">cost</a>()</pre>
 </li>
 </ul>
 <a name="costFromArray-double:A-">
@@ -374,7 +374,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>costFromArray</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1031">costFromArray</a>(double[]&nbsp;stats)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1039">costFromArray</a>(double[]&nbsp;stats)</pre>
 <div class="block">Function to compute a scaled cost using <code>org.apache.commons.math3.stat.descriptive.DescriptiveStatistics</code>.
  It assumes that this is a zero sum set of costs.  It assumes that the worst case
  possible is all of the elements in one region server and the rest having 0.</div>
@@ -392,7 +392,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockList">
 <li class="blockList">
 <h4>getSum</h4>
-<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1065">getSum</a>(double[]&nbsp;stats)</pre>
+<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1073">getSum</a>(double[]&nbsp;stats)</pre>
 </li>
 </ul>
 <a name="scale-double-double-double-">
@@ -401,7 +401,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
 <ul class="blockListLast">
 <li class="blockList">
 <h4>scale</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1081">scale</a>(double&nbsp;min,
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#line.1089">scale</a>(double&nbsp;min,
                        double&nbsp;max,
                        double&nbsp;value)</pre>
 <div class="block">Scale the value between 0 and 1.</div>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
index bcd3aa7..12fa47f 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.735">StochasticLoadBalancer.LoadCandidateGenerator</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.743">StochasticLoadBalancer.LoadCandidateGenerator</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></pre>
 </li>
 </ul>
@@ -204,7 +204,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>LoadCandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.735">LoadCandidateGenerator</a>()</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.743">LoadCandidateGenerator</a>()</pre>
 </li>
 </ul>
 </li>
@@ -221,7 +221,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>generate</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.738">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.746">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#generate-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">generate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></code></dd>
@@ -234,7 +234,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>pickLeastLoadedServer</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.746">pickLeastLoadedServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.754">pickLeastLoadedServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                   int&nbsp;thisServer)</pre>
 </li>
 </ul>
@@ -244,7 +244,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>pickMostLoadedServer</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.759">pickMostLoadedServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LoadCandidateGenerator.html#line.767">pickMostLoadedServer</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                  int&nbsp;thisServer)</pre>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
index 845fde0..51fd40d 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.773">StochasticLoadBalancer.LocalityBasedCandidateGenerator</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.781">StochasticLoadBalancer.LocalityBasedCandidateGenerator</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></pre>
 </li>
 </ul>
@@ -232,7 +232,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>masterServices</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.775">masterServices</a></pre>
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.783">masterServices</a></pre>
 </li>
 </ul>
 </li>
@@ -249,7 +249,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>LocalityBasedCandidateGenerator</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.777">LocalityBasedCandidateGenerator</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;masterServices)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.785">LocalityBasedCandidateGenerator</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;masterServices)</pre>
 </li>
 </ul>
 </li>
@@ -266,7 +266,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>generate</h4>
-<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.782">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.790">generate</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html#generate-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">generate</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CandidateGenerator</a></code></dd>
@@ -279,7 +279,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>tryMoveOrSwap</h4>
-<pre>private&nbsp;org.apache.hbase.thirdparty.com.google.common.base.Optional&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.812">tryMoveOrSwap</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>private&nbsp;org.apache.hbase.thirdparty.com.google.common.base.Optional&lt;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.Action</a>&gt;&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.820">tryMoveOrSwap</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                                                                                                    int&nbsp;fromServer,
                                                                                                                    int&nbsp;fromRegion,
                                                                                                                    int&nbsp;toServer)</pre>
@@ -293,7 +293,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>getWeightedLocality</h4>
-<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.837">getWeightedLocality</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
+<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.845">getWeightedLocality</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster,
                                    int&nbsp;region,
                                    int&nbsp;server)</pre>
 </li>
@@ -304,7 +304,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>setServices</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.841">setServices</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;services)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCandidateGenerator.html#line.849">setServices</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;services)</pre>
 </li>
 </ul>
 </li>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
index 6a82128..9e8c8eb 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1237">StochasticLoadBalancer.LocalityBasedCostFunction</a>
+<pre>abstract static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1245">StochasticLoadBalancer.LocalityBasedCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Compute a cost of a potential cluster configuration based upon where
  <a href="../../../../../../org/apache/hadoop/hbase/regionserver/HStoreFile.html" title="class in org.apache.hadoop.hbase.regionserver"><code>HStoreFile</code></a>s are located.</div>
@@ -274,7 +274,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>type</h4>
-<pre>private final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.LocalityType</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1239">type</a></pre>
+<pre>private final&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.LocalityType</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1247">type</a></pre>
 </li>
 </ul>
 <a name="bestLocality">
@@ -283,7 +283,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>bestLocality</h4>
-<pre>private&nbsp;double <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1241">bestLocality</a></pre>
+<pre>private&nbsp;double <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1249">bestLocality</a></pre>
 </li>
 </ul>
 <a name="locality">
@@ -292,7 +292,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>locality</h4>
-<pre>private&nbsp;double <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1242">locality</a></pre>
+<pre>private&nbsp;double <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1250">locality</a></pre>
 </li>
 </ul>
 <a name="services">
@@ -301,7 +301,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>services</h4>
-<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1244">services</a></pre>
+<pre>private&nbsp;<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1252">services</a></pre>
 </li>
 </ul>
 </li>
@@ -318,7 +318,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>LocalityBasedCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1246">LocalityBasedCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1254">LocalityBasedCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf,
                           <a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;srv,
                           <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster.LocalityType</a>&nbsp;type,
                           <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>&nbsp;localityCostKey,
@@ -339,7 +339,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>regionIndexToEntityIndex</h4>
-<pre>abstract&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1262">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
+<pre>abstract&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1270">regionIndexToEntityIndex</a>(int&nbsp;region)</pre>
 <div class="block">Maps region to the current entity (server or rack) on which it is stored</div>
 </li>
 </ul>
@@ -349,7 +349,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>setServices</h4>
-<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1264">setServices</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;srvc)</pre>
+<pre>public&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1272">setServices</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/MasterServices.html" title="interface in org.apache.hadoop.hbase.master">MasterServices</a>&nbsp;srvc)</pre>
 </li>
 </ul>
 <a name="init-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">
@@ -358,7 +358,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>init</h4>
-<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1269">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
+<pre>void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1277">init</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.html" title="class in org.apache.hadoop.hbase.master.balancer">BaseLoadBalancer.Cluster</a>&nbsp;cluster)</pre>
 <div class="block"><span class="descfrmTypeLabel">Description copied from class:&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#init-org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster-">StochasticLoadBalancer.CostFunction</a></code></span></div>
 <div class="block">Called once per LB invocation to give the cost function
  to initialize it's state, and perform any costly calculation.</div>
@@ -374,7 +374,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>regionMoved</h4>
-<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1291">regionMoved</a>(int&nbsp;region,
+<pre>protected&nbsp;void&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1299">regionMoved</a>(int&nbsp;region,
                            int&nbsp;oldServer,
                            int&nbsp;newServer)</pre>
 <dl>
@@ -389,7 +389,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1303">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1311">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>
@@ -402,7 +402,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>getMostLocalEntityForRegion</h4>
-<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1307">getMostLocalEntityForRegion</a>(int&nbsp;region)</pre>
+<pre>private&nbsp;int&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1315">getMostLocalEntityForRegion</a>(int&nbsp;region)</pre>
 </li>
 </ul>
 <a name="getWeightedLocality-int-int-">
@@ -411,7 +411,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getWeightedLocality</h4>
-<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1311">getWeightedLocality</a>(int&nbsp;region,
+<pre>private&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.LocalityBasedCostFunction.html#line.1319">getWeightedLocality</a>(int&nbsp;region,
                                    int&nbsp;entity)</pre>
 </li>
 </ul>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
index 0302678..33760da 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html
@@ -128,7 +128,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1683">StochasticLoadBalancer.MemStoreSizeCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1691">StochasticLoadBalancer.MemStoreSizeCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadAsRateFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadAsRateFunction</a></pre>
 <div class="block">Compute the cost of total memstore size.  The more unbalanced the higher the
  computed cost will be.  This uses a rolling average of regionload.</div>
@@ -250,7 +250,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>MEMSTORE_SIZE_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1685">MEMSTORE_SIZE_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1693">MEMSTORE_SIZE_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MemStoreSizeCostFunction.MEMSTORE_SIZE_COST_KEY">Constant Field Values</a></dd>
@@ -263,7 +263,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>DEFAULT_MEMSTORE_SIZE_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1687">DEFAULT_MEMSTORE_SIZE_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1695">DEFAULT_MEMSTORE_SIZE_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MemStoreSizeCostFunction.DEFAULT_MEMSTORE_SIZE_COST">Constant Field Values</a></dd>
@@ -284,7 +284,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>MemStoreSizeCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1689">MemStoreSizeCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1697">MemStoreSizeCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -301,7 +301,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>getCostFromRl</h4>
-<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1695">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
+<pre>protected&nbsp;double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MemStoreSizeCostFunction.html#line.1703">getCostFromRl</a>(<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.html" title="class in org.apache.hadoop.hbase.master.balancer">BalancerRegionLoad</a>&nbsp;rl)</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html#getCostFromRl-org.apache.hadoop.hbase.master.balancer.BalancerRegionLoad-">getCostFromRl</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFromRegionLoadFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFromRegionLoadFunction</a></code></dd>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
index fbc6f3a..69e95d3 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html
@@ -118,7 +118,7 @@ var activeTableTab = "activeTableTab";
 </dl>
 <hr>
 <br>
-<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1095">StochasticLoadBalancer.MoveCostFunction</a>
+<pre>static class <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html#line.1103">StochasticLoadBalancer.MoveCostFunction</a>
 extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></pre>
 <div class="block">Given the starting state of the regions and a potential ending state
  compute cost based upon the number of regions that have moved.</div>
@@ -242,7 +242,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>MOVE_COST_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1096">MOVE_COST_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1104">MOVE_COST_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MoveCostFunction.MOVE_COST_KEY">Constant Field Values</a></dd>
@@ -255,7 +255,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>MAX_MOVES_PERCENT_KEY</h4>
-<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1097">MAX_MOVES_PERCENT_KEY</a></pre>
+<pre>private static final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1105">MAX_MOVES_PERCENT_KEY</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MoveCostFunction.MAX_MOVES_PERCENT_KEY">Constant Field Values</a></dd>
@@ -268,7 +268,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_MOVE_COST</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1099">DEFAULT_MOVE_COST</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1107">DEFAULT_MOVE_COST</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST">Constant Field Values</a></dd>
@@ -281,7 +281,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_MAX_MOVES</h4>
-<pre>private static final&nbsp;int <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1100">DEFAULT_MAX_MOVES</a></pre>
+<pre>private static final&nbsp;int <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1108">DEFAULT_MAX_MOVES</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MoveCostFunction.DEFAULT_MAX_MOVES">Constant Field Values</a></dd>
@@ -294,7 +294,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockList">
 <li class="blockList">
 <h4>DEFAULT_MAX_MOVE_PERCENT</h4>
-<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1101">DEFAULT_MAX_MOVE_PERCENT</a></pre>
+<pre>private static final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1109">DEFAULT_MAX_MOVE_PERCENT</a></pre>
 <dl>
 <dt><span class="seeLabel">See Also:</span></dt>
 <dd><a href="../../../../../../constant-values.html#org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer.MoveCostFunction.DEFAULT_MAX_MOVE_PERCENT">Constant Field Values</a></dd>
@@ -307,7 +307,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>maxMovesPercent</h4>
-<pre>private final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1103">maxMovesPercent</a></pre>
+<pre>private final&nbsp;float <a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1111">maxMovesPercent</a></pre>
 </li>
 </ul>
 </li>
@@ -324,7 +324,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>MoveCostFunction</h4>
-<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1105">MoveCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
+<pre><a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1113">MoveCostFunction</a>(org.apache.hadoop.conf.Configuration&nbsp;conf)</pre>
 </li>
 </ul>
 </li>
@@ -341,7 +341,7 @@ extends <a href="../../../../../../org/apache/hadoop/hbase/master/balancer/Stoch
 <ul class="blockListLast">
 <li class="blockList">
 <h4>cost</h4>
-<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1116">cost</a>()</pre>
+<pre>double&nbsp;<a href="../../../../../../src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.MoveCostFunction.html#line.1124">cost</a>()</pre>
 <dl>
 <dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
 <dd><code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html#cost--">cost</a></code>&nbsp;in class&nbsp;<code><a href="../../../../../../org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CostFunction.html" title="class in org.apache.hadoop.hbase.master.balancer">StochasticLoadBalancer.CostFunction</a></code></dd>


[42/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html
index 1814633..02f25e8 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/CatalogJanitor.html
@@ -119,355 +119,357 @@
 <span class="sourceLineNo">111</span>  protected void chore() {<a name="line.111"></a>
 <span class="sourceLineNo">112</span>    try {<a name="line.112"></a>
 <span class="sourceLineNo">113</span>      AssignmentManager am = this.services.getAssignmentManager();<a name="line.113"></a>
-<span class="sourceLineNo">114</span>      if (this.enabled.get() &amp;&amp; !this.services.isInMaintenanceMode() &amp;&amp; am != null &amp;&amp;<a name="line.114"></a>
-<span class="sourceLineNo">115</span>        am.isMetaLoaded() &amp;&amp; !am.hasRegionsInTransition()) {<a name="line.115"></a>
-<span class="sourceLineNo">116</span>        scan();<a name="line.116"></a>
-<span class="sourceLineNo">117</span>      } else {<a name="line.117"></a>
-<span class="sourceLineNo">118</span>        LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() +<a name="line.118"></a>
-<span class="sourceLineNo">119</span>          ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am +<a name="line.119"></a>
-<span class="sourceLineNo">120</span>          ", metaLoaded=" + (am != null &amp;&amp; am.isMetaLoaded()) + ", hasRIT=" +<a name="line.120"></a>
-<span class="sourceLineNo">121</span>          (am != null &amp;&amp; am.hasRegionsInTransition()));<a name="line.121"></a>
-<span class="sourceLineNo">122</span>      }<a name="line.122"></a>
-<span class="sourceLineNo">123</span>    } catch (IOException e) {<a name="line.123"></a>
-<span class="sourceLineNo">124</span>      LOG.warn("Failed scan of catalog table", e);<a name="line.124"></a>
-<span class="sourceLineNo">125</span>    }<a name="line.125"></a>
-<span class="sourceLineNo">126</span>  }<a name="line.126"></a>
-<span class="sourceLineNo">127</span><a name="line.127"></a>
-<span class="sourceLineNo">128</span>  /**<a name="line.128"></a>
-<span class="sourceLineNo">129</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.129"></a>
-<span class="sourceLineNo">130</span>   * regions, and an ordered map of split parents.<a name="line.130"></a>
-<span class="sourceLineNo">131</span>   * @return triple of scanned rows, map of merged regions and map of split<a name="line.131"></a>
-<span class="sourceLineNo">132</span>   *         parent regioninfos<a name="line.132"></a>
-<span class="sourceLineNo">133</span>   * @throws IOException<a name="line.133"></a>
-<span class="sourceLineNo">134</span>   */<a name="line.134"></a>
-<span class="sourceLineNo">135</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.135"></a>
-<span class="sourceLineNo">136</span>    getMergedRegionsAndSplitParents() throws IOException {<a name="line.136"></a>
-<span class="sourceLineNo">137</span>    return getMergedRegionsAndSplitParents(null);<a name="line.137"></a>
-<span class="sourceLineNo">138</span>  }<a name="line.138"></a>
-<span class="sourceLineNo">139</span><a name="line.139"></a>
-<span class="sourceLineNo">140</span>  /**<a name="line.140"></a>
-<span class="sourceLineNo">141</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.141"></a>
-<span class="sourceLineNo">142</span>   * regions, and an ordered map of split parents. if the given table name is<a name="line.142"></a>
-<span class="sourceLineNo">143</span>   * null, return merged regions and split parents of all tables, else only the<a name="line.143"></a>
-<span class="sourceLineNo">144</span>   * specified table<a name="line.144"></a>
-<span class="sourceLineNo">145</span>   * @param tableName null represents all tables<a name="line.145"></a>
-<span class="sourceLineNo">146</span>   * @return triple of scanned rows, and map of merged regions, and map of split<a name="line.146"></a>
-<span class="sourceLineNo">147</span>   *         parent regioninfos<a name="line.147"></a>
-<span class="sourceLineNo">148</span>   * @throws IOException<a name="line.148"></a>
-<span class="sourceLineNo">149</span>   */<a name="line.149"></a>
-<span class="sourceLineNo">150</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.150"></a>
-<span class="sourceLineNo">151</span>    getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {<a name="line.151"></a>
-<span class="sourceLineNo">152</span>    final boolean isTableSpecified = (tableName != null);<a name="line.152"></a>
-<span class="sourceLineNo">153</span>    // TODO: Only works with single hbase:meta region currently.  Fix.<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    final AtomicInteger count = new AtomicInteger(0);<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    // Keep Map of found split parents.  There are candidates for cleanup.<a name="line.155"></a>
-<span class="sourceLineNo">156</span>    // Use a comparator that has split parents come before its daughters.<a name="line.156"></a>
-<span class="sourceLineNo">157</span>    final Map&lt;RegionInfo, Result&gt; splitParents = new TreeMap&lt;&gt;(new SplitParentFirstComparator());<a name="line.157"></a>
-<span class="sourceLineNo">158</span>    final Map&lt;RegionInfo, Result&gt; mergedRegions = new TreeMap&lt;&gt;(RegionInfo.COMPARATOR);<a name="line.158"></a>
-<span class="sourceLineNo">159</span>    // This visitor collects split parents and counts rows in the hbase:meta table<a name="line.159"></a>
-<span class="sourceLineNo">160</span><a name="line.160"></a>
-<span class="sourceLineNo">161</span>    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {<a name="line.161"></a>
-<span class="sourceLineNo">162</span>      @Override<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      public boolean visit(Result r) throws IOException {<a name="line.163"></a>
-<span class="sourceLineNo">164</span>        if (r == null || r.isEmpty()) return true;<a name="line.164"></a>
-<span class="sourceLineNo">165</span>        count.incrementAndGet();<a name="line.165"></a>
-<span class="sourceLineNo">166</span>        RegionInfo info = MetaTableAccessor.getRegionInfo(r);<a name="line.166"></a>
-<span class="sourceLineNo">167</span>        if (info == null) return true; // Keep scanning<a name="line.167"></a>
-<span class="sourceLineNo">168</span>        if (isTableSpecified<a name="line.168"></a>
-<span class="sourceLineNo">169</span>            &amp;&amp; info.getTable().compareTo(tableName) &gt; 0) {<a name="line.169"></a>
-<span class="sourceLineNo">170</span>          // Another table, stop scanning<a name="line.170"></a>
-<span class="sourceLineNo">171</span>          return false;<a name="line.171"></a>
-<span class="sourceLineNo">172</span>        }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>        if (LOG.isTraceEnabled()) LOG.trace("" + info + " IS-SPLIT_PARENT=" + info.isSplitParent());<a name="line.173"></a>
-<span class="sourceLineNo">174</span>        if (info.isSplitParent()) splitParents.put(info, r);<a name="line.174"></a>
-<span class="sourceLineNo">175</span>        if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span>          mergedRegions.put(info, r);<a name="line.176"></a>
-<span class="sourceLineNo">177</span>        }<a name="line.177"></a>
-<span class="sourceLineNo">178</span>        // Returning true means "keep scanning"<a name="line.178"></a>
-<span class="sourceLineNo">179</span>        return true;<a name="line.179"></a>
-<span class="sourceLineNo">180</span>      }<a name="line.180"></a>
-<span class="sourceLineNo">181</span>    };<a name="line.181"></a>
-<span class="sourceLineNo">182</span><a name="line.182"></a>
-<span class="sourceLineNo">183</span>    // Run full scan of hbase:meta catalog table passing in our custom visitor with<a name="line.183"></a>
-<span class="sourceLineNo">184</span>    // the start row<a name="line.184"></a>
-<span class="sourceLineNo">185</span>    MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span>    return new Triple&lt;&gt;(count.get(), mergedRegions, splitParents);<a name="line.187"></a>
-<span class="sourceLineNo">188</span>  }<a name="line.188"></a>
-<span class="sourceLineNo">189</span><a name="line.189"></a>
-<span class="sourceLineNo">190</span>  /**<a name="line.190"></a>
-<span class="sourceLineNo">191</span>   * If merged region no longer holds reference to the merge regions, archive<a name="line.191"></a>
-<span class="sourceLineNo">192</span>   * merge region on hdfs and perform deleting references in hbase:meta<a name="line.192"></a>
-<span class="sourceLineNo">193</span>   * @param mergedRegion<a name="line.193"></a>
-<span class="sourceLineNo">194</span>   * @return true if we delete references in merged region on hbase:meta and archive<a name="line.194"></a>
-<span class="sourceLineNo">195</span>   *         the files on the file system<a name="line.195"></a>
-<span class="sourceLineNo">196</span>   * @throws IOException<a name="line.196"></a>
-<span class="sourceLineNo">197</span>   */<a name="line.197"></a>
-<span class="sourceLineNo">198</span>  boolean cleanMergeRegion(final RegionInfo mergedRegion,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>      final RegionInfo regionA, final RegionInfo regionB) throws IOException {<a name="line.199"></a>
-<span class="sourceLineNo">200</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.200"></a>
-<span class="sourceLineNo">201</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.201"></a>
-<span class="sourceLineNo">202</span>    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());<a name="line.202"></a>
-<span class="sourceLineNo">203</span>    TableDescriptor htd = getTableDescriptor(mergedRegion.getTable());<a name="line.203"></a>
-<span class="sourceLineNo">204</span>    HRegionFileSystem regionFs = null;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>    try {<a name="line.205"></a>
-<span class="sourceLineNo">206</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.206"></a>
-<span class="sourceLineNo">207</span>          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);<a name="line.207"></a>
-<span class="sourceLineNo">208</span>    } catch (IOException e) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span>      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());<a name="line.209"></a>
-<span class="sourceLineNo">210</span>    }<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    if (regionFs == null || !regionFs.hasReferences(htd)) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span>      LOG.debug("Deleting region " + regionA.getShortNameToLog() + " and "<a name="line.212"></a>
-<span class="sourceLineNo">213</span>          + regionB.getShortNameToLog()<a name="line.213"></a>
-<span class="sourceLineNo">214</span>          + " from fs because merged region no longer holds references");<a name="line.214"></a>
-<span class="sourceLineNo">215</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),<a name="line.216"></a>
-<span class="sourceLineNo">217</span>          mergedRegion, regionA, regionB));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>      // Remove from in-memory states<a name="line.218"></a>
-<span class="sourceLineNo">219</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);<a name="line.219"></a>
-<span class="sourceLineNo">220</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);<a name="line.220"></a>
-<span class="sourceLineNo">221</span>      this.services.getServerManager().removeRegion(regionA);<a name="line.221"></a>
-<span class="sourceLineNo">222</span>      this.services.getServerManager().removeRegion(regionB);<a name="line.222"></a>
-<span class="sourceLineNo">223</span>      return true;<a name="line.223"></a>
-<span class="sourceLineNo">224</span>    }<a name="line.224"></a>
-<span class="sourceLineNo">225</span>    return false;<a name="line.225"></a>
-<span class="sourceLineNo">226</span>  }<a name="line.226"></a>
-<span class="sourceLineNo">227</span><a name="line.227"></a>
-<span class="sourceLineNo">228</span>  /**<a name="line.228"></a>
-<span class="sourceLineNo">229</span>   * Run janitorial scan of catalog &lt;code&gt;hbase:meta&lt;/code&gt; table looking for<a name="line.229"></a>
-<span class="sourceLineNo">230</span>   * garbage to collect.<a name="line.230"></a>
-<span class="sourceLineNo">231</span>   * @return number of archiving jobs started.<a name="line.231"></a>
-<span class="sourceLineNo">232</span>   * @throws IOException<a name="line.232"></a>
-<span class="sourceLineNo">233</span>   */<a name="line.233"></a>
-<span class="sourceLineNo">234</span>  int scan() throws IOException {<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    int result = 0;<a name="line.235"></a>
-<span class="sourceLineNo">236</span><a name="line.236"></a>
-<span class="sourceLineNo">237</span>    try {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>      if (!alreadyRunning.compareAndSet(false, true)) {<a name="line.238"></a>
-<span class="sourceLineNo">239</span>        LOG.debug("CatalogJanitor already running");<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        return result;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>      Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt; scanTriple =<a name="line.242"></a>
-<span class="sourceLineNo">243</span>        getMergedRegionsAndSplitParents();<a name="line.243"></a>
-<span class="sourceLineNo">244</span>      /**<a name="line.244"></a>
-<span class="sourceLineNo">245</span>       * clean merge regions first<a name="line.245"></a>
-<span class="sourceLineNo">246</span>       */<a name="line.246"></a>
-<span class="sourceLineNo">247</span>      Map&lt;RegionInfo, Result&gt; mergedRegions = scanTriple.getSecond();<a name="line.247"></a>
-<span class="sourceLineNo">248</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : mergedRegions.entrySet()) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span>        if (this.services.isInMaintenanceMode()) {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>          // Stop cleaning if the master is in maintenance mode<a name="line.250"></a>
-<span class="sourceLineNo">251</span>          break;<a name="line.251"></a>
-<span class="sourceLineNo">252</span>        }<a name="line.252"></a>
-<span class="sourceLineNo">253</span><a name="line.253"></a>
-<span class="sourceLineNo">254</span>        PairOfSameType&lt;RegionInfo&gt; p = MetaTableAccessor.getMergeRegions(e.getValue());<a name="line.254"></a>
-<span class="sourceLineNo">255</span>        RegionInfo regionA = p.getFirst();<a name="line.255"></a>
-<span class="sourceLineNo">256</span>        RegionInfo regionB = p.getSecond();<a name="line.256"></a>
-<span class="sourceLineNo">257</span>        if (regionA == null || regionB == null) {<a name="line.257"></a>
-<span class="sourceLineNo">258</span>          LOG.warn("Unexpected references regionA="<a name="line.258"></a>
-<span class="sourceLineNo">259</span>              + (regionA == null ? "null" : regionA.getShortNameToLog())<a name="line.259"></a>
-<span class="sourceLineNo">260</span>              + ",regionB="<a name="line.260"></a>
-<span class="sourceLineNo">261</span>              + (regionB == null ? "null" : regionB.getShortNameToLog())<a name="line.261"></a>
-<span class="sourceLineNo">262</span>              + " in merged region " + e.getKey().getShortNameToLog());<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        } else {<a name="line.263"></a>
-<span class="sourceLineNo">264</span>          if (cleanMergeRegion(e.getKey(), regionA, regionB)) {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>            result++;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>          }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>        }<a name="line.267"></a>
-<span class="sourceLineNo">268</span>      }<a name="line.268"></a>
-<span class="sourceLineNo">269</span>      /**<a name="line.269"></a>
-<span class="sourceLineNo">270</span>       * clean split parents<a name="line.270"></a>
-<span class="sourceLineNo">271</span>       */<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      Map&lt;RegionInfo, Result&gt; splitParents = scanTriple.getThird();<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span>      // Now work on our list of found parents. See if any we can clean up.<a name="line.274"></a>
-<span class="sourceLineNo">275</span>      // regions whose parents are still around<a name="line.275"></a>
-<span class="sourceLineNo">276</span>      HashSet&lt;String&gt; parentNotCleaned = new HashSet&lt;&gt;();<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : splitParents.entrySet()) {<a name="line.277"></a>
-<span class="sourceLineNo">278</span>        if (this.services.isInMaintenanceMode()) {<a name="line.278"></a>
-<span class="sourceLineNo">279</span>          // Stop cleaning if the master is in maintenance mode<a name="line.279"></a>
-<span class="sourceLineNo">280</span>          break;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>        }<a name="line.281"></a>
-<span class="sourceLineNo">282</span><a name="line.282"></a>
-<span class="sourceLineNo">283</span>        if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &amp;&amp;<a name="line.283"></a>
-<span class="sourceLineNo">284</span>            cleanParent(e.getKey(), e.getValue())) {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>          result++;<a name="line.285"></a>
-<span class="sourceLineNo">286</span>        } else {<a name="line.286"></a>
-<span class="sourceLineNo">287</span>          // We could not clean the parent, so it's daughters should not be<a name="line.287"></a>
-<span class="sourceLineNo">288</span>          // cleaned either (HBASE-6160)<a name="line.288"></a>
-<span class="sourceLineNo">289</span>          PairOfSameType&lt;RegionInfo&gt; daughters =<a name="line.289"></a>
-<span class="sourceLineNo">290</span>              MetaTableAccessor.getDaughterRegions(e.getValue());<a name="line.290"></a>
-<span class="sourceLineNo">291</span>          parentNotCleaned.add(daughters.getFirst().getEncodedName());<a name="line.291"></a>
-<span class="sourceLineNo">292</span>          parentNotCleaned.add(daughters.getSecond().getEncodedName());<a name="line.292"></a>
-<span class="sourceLineNo">293</span>        }<a name="line.293"></a>
-<span class="sourceLineNo">294</span>      }<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      return result;<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    } finally {<a name="line.296"></a>
-<span class="sourceLineNo">297</span>      alreadyRunning.set(false);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>    }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>  }<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span>  /**<a name="line.301"></a>
-<span class="sourceLineNo">302</span>   * Compare HRegionInfos in a way that has split parents sort BEFORE their<a name="line.302"></a>
-<span class="sourceLineNo">303</span>   * daughters.<a name="line.303"></a>
-<span class="sourceLineNo">304</span>   */<a name="line.304"></a>
-<span class="sourceLineNo">305</span>  static class SplitParentFirstComparator implements Comparator&lt;RegionInfo&gt; {<a name="line.305"></a>
-<span class="sourceLineNo">306</span>    Comparator&lt;byte[]&gt; rowEndKeyComparator = new Bytes.RowEndKeyComparator();<a name="line.306"></a>
-<span class="sourceLineNo">307</span>    @Override<a name="line.307"></a>
-<span class="sourceLineNo">308</span>    public int compare(RegionInfo left, RegionInfo right) {<a name="line.308"></a>
-<span class="sourceLineNo">309</span>      // This comparator differs from the one RegionInfo in that it sorts<a name="line.309"></a>
-<span class="sourceLineNo">310</span>      // parent before daughters.<a name="line.310"></a>
-<span class="sourceLineNo">311</span>      if (left == null) return -1;<a name="line.311"></a>
-<span class="sourceLineNo">312</span>      if (right == null) return 1;<a name="line.312"></a>
-<span class="sourceLineNo">313</span>      // Same table name.<a name="line.313"></a>
-<span class="sourceLineNo">314</span>      int result = left.getTable().compareTo(right.getTable());<a name="line.314"></a>
-<span class="sourceLineNo">315</span>      if (result != 0) return result;<a name="line.315"></a>
-<span class="sourceLineNo">316</span>      // Compare start keys.<a name="line.316"></a>
-<span class="sourceLineNo">317</span>      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());<a name="line.317"></a>
-<span class="sourceLineNo">318</span>      if (result != 0) return result;<a name="line.318"></a>
-<span class="sourceLineNo">319</span>      // Compare end keys, but flip the operands so parent comes first<a name="line.319"></a>
-<span class="sourceLineNo">320</span>      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());<a name="line.320"></a>
-<span class="sourceLineNo">321</span><a name="line.321"></a>
-<span class="sourceLineNo">322</span>      return result;<a name="line.322"></a>
-<span class="sourceLineNo">323</span>    }<a name="line.323"></a>
-<span class="sourceLineNo">324</span>  }<a name="line.324"></a>
-<span class="sourceLineNo">325</span><a name="line.325"></a>
-<span class="sourceLineNo">326</span>  /**<a name="line.326"></a>
-<span class="sourceLineNo">327</span>   * If daughters no longer hold reference to the parents, delete the parent.<a name="line.327"></a>
-<span class="sourceLineNo">328</span>   * @param parent RegionInfo of split offlined parent<a name="line.328"></a>
-<span class="sourceLineNo">329</span>   * @param rowContent Content of &lt;code&gt;parent&lt;/code&gt; row in<a name="line.329"></a>
-<span class="sourceLineNo">330</span>   * &lt;code&gt;metaRegionName&lt;/code&gt;<a name="line.330"></a>
-<span class="sourceLineNo">331</span>   * @return True if we removed &lt;code&gt;parent&lt;/code&gt; from meta table and from<a name="line.331"></a>
-<span class="sourceLineNo">332</span>   * the filesystem.<a name="line.332"></a>
-<span class="sourceLineNo">333</span>   * @throws IOException<a name="line.333"></a>
-<span class="sourceLineNo">334</span>   */<a name="line.334"></a>
-<span class="sourceLineNo">335</span>  boolean cleanParent(final RegionInfo parent, Result rowContent)<a name="line.335"></a>
-<span class="sourceLineNo">336</span>  throws IOException {<a name="line.336"></a>
-<span class="sourceLineNo">337</span>    // Check whether it is a merged region and not clean reference<a name="line.337"></a>
-<span class="sourceLineNo">338</span>    // No necessary to check MERGEB_QUALIFIER because these two qualifiers will<a name="line.338"></a>
-<span class="sourceLineNo">339</span>    // be inserted/deleted together<a name="line.339"></a>
-<span class="sourceLineNo">340</span>    if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span>      // wait cleaning merge region first<a name="line.341"></a>
-<span class="sourceLineNo">342</span>      return false;<a name="line.342"></a>
-<span class="sourceLineNo">343</span>    }<a name="line.343"></a>
-<span class="sourceLineNo">344</span>    // Run checks on each daughter split.<a name="line.344"></a>
-<span class="sourceLineNo">345</span>    PairOfSameType&lt;RegionInfo&gt; daughters = MetaTableAccessor.getDaughterRegions(rowContent);<a name="line.345"></a>
-<span class="sourceLineNo">346</span>    Pair&lt;Boolean, Boolean&gt; a = checkDaughterInFs(parent, daughters.getFirst());<a name="line.346"></a>
-<span class="sourceLineNo">347</span>    Pair&lt;Boolean, Boolean&gt; b = checkDaughterInFs(parent, daughters.getSecond());<a name="line.347"></a>
-<span class="sourceLineNo">348</span>    if (hasNoReferences(a) &amp;&amp; hasNoReferences(b)) {<a name="line.348"></a>
-<span class="sourceLineNo">349</span>      String daughterA = daughters.getFirst() != null?<a name="line.349"></a>
-<span class="sourceLineNo">350</span>          daughters.getFirst().getShortNameToLog(): "null";<a name="line.350"></a>
-<span class="sourceLineNo">351</span>      String daughterB = daughters.getSecond() != null?<a name="line.351"></a>
-<span class="sourceLineNo">352</span>          daughters.getSecond().getShortNameToLog(): "null";<a name="line.352"></a>
-<span class="sourceLineNo">353</span>      LOG.debug("Deleting region " + parent.getShortNameToLog() +<a name="line.353"></a>
-<span class="sourceLineNo">354</span>        " because daughters -- " + daughterA + ", " + daughterB +<a name="line.354"></a>
-<span class="sourceLineNo">355</span>        " -- no longer hold references");<a name="line.355"></a>
-<span class="sourceLineNo">356</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.356"></a>
-<span class="sourceLineNo">357</span>      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));<a name="line.357"></a>
-<span class="sourceLineNo">358</span>      // Remove from in-memory states<a name="line.358"></a>
-<span class="sourceLineNo">359</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);<a name="line.359"></a>
-<span class="sourceLineNo">360</span>      this.services.getServerManager().removeRegion(parent);<a name="line.360"></a>
-<span class="sourceLineNo">361</span>      return true;<a name="line.361"></a>
-<span class="sourceLineNo">362</span>    }<a name="line.362"></a>
-<span class="sourceLineNo">363</span>    return false;<a name="line.363"></a>
-<span class="sourceLineNo">364</span>  }<a name="line.364"></a>
-<span class="sourceLineNo">365</span><a name="line.365"></a>
-<span class="sourceLineNo">366</span>  /**<a name="line.366"></a>
-<span class="sourceLineNo">367</span>   * @param p A pair where the first boolean says whether or not the daughter<a name="line.367"></a>
-<span class="sourceLineNo">368</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.368"></a>
-<span class="sourceLineNo">369</span>   * whether the daughter has references to the parent.<a name="line.369"></a>
-<span class="sourceLineNo">370</span>   * @return True the passed &lt;code&gt;p&lt;/code&gt; signifies no references.<a name="line.370"></a>
-<span class="sourceLineNo">371</span>   */<a name="line.371"></a>
-<span class="sourceLineNo">372</span>  private boolean hasNoReferences(final Pair&lt;Boolean, Boolean&gt; p) {<a name="line.372"></a>
-<span class="sourceLineNo">373</span>    return !p.getFirst() || !p.getSecond();<a name="line.373"></a>
-<span class="sourceLineNo">374</span>  }<a name="line.374"></a>
-<span class="sourceLineNo">375</span><a name="line.375"></a>
-<span class="sourceLineNo">376</span>  /**<a name="line.376"></a>
-<span class="sourceLineNo">377</span>   * Checks if a daughter region -- either splitA or splitB -- still holds<a name="line.377"></a>
-<span class="sourceLineNo">378</span>   * references to parent.<a name="line.378"></a>
-<span class="sourceLineNo">379</span>   * @param parent Parent region<a name="line.379"></a>
-<span class="sourceLineNo">380</span>   * @param daughter Daughter region<a name="line.380"></a>
-<span class="sourceLineNo">381</span>   * @return A pair where the first boolean says whether or not the daughter<a name="line.381"></a>
-<span class="sourceLineNo">382</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.382"></a>
-<span class="sourceLineNo">383</span>   * whether the daughter has references to the parent.<a name="line.383"></a>
-<span class="sourceLineNo">384</span>   * @throws IOException<a name="line.384"></a>
-<span class="sourceLineNo">385</span>   */<a name="line.385"></a>
-<span class="sourceLineNo">386</span>  Pair&lt;Boolean, Boolean&gt; checkDaughterInFs(final RegionInfo parent, final RegionInfo daughter)<a name="line.386"></a>
-<span class="sourceLineNo">387</span>  throws IOException {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>    if (daughter == null)  {<a name="line.388"></a>
-<span class="sourceLineNo">389</span>      return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span><a name="line.391"></a>
-<span class="sourceLineNo">392</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.392"></a>
-<span class="sourceLineNo">393</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.393"></a>
-<span class="sourceLineNo">394</span>    Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());<a name="line.394"></a>
-<span class="sourceLineNo">395</span><a name="line.395"></a>
-<span class="sourceLineNo">396</span>    Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());<a name="line.396"></a>
+<span class="sourceLineNo">114</span>      if (this.enabled.get() &amp;&amp; !this.services.isInMaintenanceMode() &amp;&amp;<a name="line.114"></a>
+<span class="sourceLineNo">115</span>        !this.services.getServerManager().isClusterShutdown() &amp;&amp; am != null &amp;&amp;<a name="line.115"></a>
+<span class="sourceLineNo">116</span>        am.isMetaLoaded() &amp;&amp; !am.hasRegionsInTransition()) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span>        scan();<a name="line.117"></a>
+<span class="sourceLineNo">118</span>      } else {<a name="line.118"></a>
+<span class="sourceLineNo">119</span>        LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() +<a name="line.119"></a>
+<span class="sourceLineNo">120</span>          ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am +<a name="line.120"></a>
+<span class="sourceLineNo">121</span>          ", metaLoaded=" + (am != null &amp;&amp; am.isMetaLoaded()) + ", hasRIT=" +<a name="line.121"></a>
+<span class="sourceLineNo">122</span>          (am != null &amp;&amp; am.hasRegionsInTransition()) + " clusterShutDown=" + this.services<a name="line.122"></a>
+<span class="sourceLineNo">123</span>          .getServerManager().isClusterShutdown());<a name="line.123"></a>
+<span class="sourceLineNo">124</span>      }<a name="line.124"></a>
+<span class="sourceLineNo">125</span>    } catch (IOException e) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span>      LOG.warn("Failed scan of catalog table", e);<a name="line.126"></a>
+<span class="sourceLineNo">127</span>    }<a name="line.127"></a>
+<span class="sourceLineNo">128</span>  }<a name="line.128"></a>
+<span class="sourceLineNo">129</span><a name="line.129"></a>
+<span class="sourceLineNo">130</span>  /**<a name="line.130"></a>
+<span class="sourceLineNo">131</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.131"></a>
+<span class="sourceLineNo">132</span>   * regions, and an ordered map of split parents.<a name="line.132"></a>
+<span class="sourceLineNo">133</span>   * @return triple of scanned rows, map of merged regions and map of split<a name="line.133"></a>
+<span class="sourceLineNo">134</span>   *         parent regioninfos<a name="line.134"></a>
+<span class="sourceLineNo">135</span>   * @throws IOException<a name="line.135"></a>
+<span class="sourceLineNo">136</span>   */<a name="line.136"></a>
+<span class="sourceLineNo">137</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.137"></a>
+<span class="sourceLineNo">138</span>    getMergedRegionsAndSplitParents() throws IOException {<a name="line.138"></a>
+<span class="sourceLineNo">139</span>    return getMergedRegionsAndSplitParents(null);<a name="line.139"></a>
+<span class="sourceLineNo">140</span>  }<a name="line.140"></a>
+<span class="sourceLineNo">141</span><a name="line.141"></a>
+<span class="sourceLineNo">142</span>  /**<a name="line.142"></a>
+<span class="sourceLineNo">143</span>   * Scans hbase:meta and returns a number of scanned rows, and a map of merged<a name="line.143"></a>
+<span class="sourceLineNo">144</span>   * regions, and an ordered map of split parents. if the given table name is<a name="line.144"></a>
+<span class="sourceLineNo">145</span>   * null, return merged regions and split parents of all tables, else only the<a name="line.145"></a>
+<span class="sourceLineNo">146</span>   * specified table<a name="line.146"></a>
+<span class="sourceLineNo">147</span>   * @param tableName null represents all tables<a name="line.147"></a>
+<span class="sourceLineNo">148</span>   * @return triple of scanned rows, and map of merged regions, and map of split<a name="line.148"></a>
+<span class="sourceLineNo">149</span>   *         parent regioninfos<a name="line.149"></a>
+<span class="sourceLineNo">150</span>   * @throws IOException<a name="line.150"></a>
+<span class="sourceLineNo">151</span>   */<a name="line.151"></a>
+<span class="sourceLineNo">152</span>  Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt;<a name="line.152"></a>
+<span class="sourceLineNo">153</span>    getMergedRegionsAndSplitParents(final TableName tableName) throws IOException {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>    final boolean isTableSpecified = (tableName != null);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>    // TODO: Only works with single hbase:meta region currently.  Fix.<a name="line.155"></a>
+<span class="sourceLineNo">156</span>    final AtomicInteger count = new AtomicInteger(0);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    // Keep Map of found split parents.  There are candidates for cleanup.<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    // Use a comparator that has split parents come before its daughters.<a name="line.158"></a>
+<span class="sourceLineNo">159</span>    final Map&lt;RegionInfo, Result&gt; splitParents = new TreeMap&lt;&gt;(new SplitParentFirstComparator());<a name="line.159"></a>
+<span class="sourceLineNo">160</span>    final Map&lt;RegionInfo, Result&gt; mergedRegions = new TreeMap&lt;&gt;(RegionInfo.COMPARATOR);<a name="line.160"></a>
+<span class="sourceLineNo">161</span>    // This visitor collects split parents and counts rows in the hbase:meta table<a name="line.161"></a>
+<span class="sourceLineNo">162</span><a name="line.162"></a>
+<span class="sourceLineNo">163</span>    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {<a name="line.163"></a>
+<span class="sourceLineNo">164</span>      @Override<a name="line.164"></a>
+<span class="sourceLineNo">165</span>      public boolean visit(Result r) throws IOException {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>        if (r == null || r.isEmpty()) return true;<a name="line.166"></a>
+<span class="sourceLineNo">167</span>        count.incrementAndGet();<a name="line.167"></a>
+<span class="sourceLineNo">168</span>        RegionInfo info = MetaTableAccessor.getRegionInfo(r);<a name="line.168"></a>
+<span class="sourceLineNo">169</span>        if (info == null) return true; // Keep scanning<a name="line.169"></a>
+<span class="sourceLineNo">170</span>        if (isTableSpecified<a name="line.170"></a>
+<span class="sourceLineNo">171</span>            &amp;&amp; info.getTable().compareTo(tableName) &gt; 0) {<a name="line.171"></a>
+<span class="sourceLineNo">172</span>          // Another table, stop scanning<a name="line.172"></a>
+<span class="sourceLineNo">173</span>          return false;<a name="line.173"></a>
+<span class="sourceLineNo">174</span>        }<a name="line.174"></a>
+<span class="sourceLineNo">175</span>        if (LOG.isTraceEnabled()) LOG.trace("" + info + " IS-SPLIT_PARENT=" + info.isSplitParent());<a name="line.175"></a>
+<span class="sourceLineNo">176</span>        if (info.isSplitParent()) splitParents.put(info, r);<a name="line.176"></a>
+<span class="sourceLineNo">177</span>        if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span>          mergedRegions.put(info, r);<a name="line.178"></a>
+<span class="sourceLineNo">179</span>        }<a name="line.179"></a>
+<span class="sourceLineNo">180</span>        // Returning true means "keep scanning"<a name="line.180"></a>
+<span class="sourceLineNo">181</span>        return true;<a name="line.181"></a>
+<span class="sourceLineNo">182</span>      }<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    };<a name="line.183"></a>
+<span class="sourceLineNo">184</span><a name="line.184"></a>
+<span class="sourceLineNo">185</span>    // Run full scan of hbase:meta catalog table passing in our custom visitor with<a name="line.185"></a>
+<span class="sourceLineNo">186</span>    // the start row<a name="line.186"></a>
+<span class="sourceLineNo">187</span>    MetaTableAccessor.scanMetaForTableRegions(this.connection, visitor, tableName);<a name="line.187"></a>
+<span class="sourceLineNo">188</span><a name="line.188"></a>
+<span class="sourceLineNo">189</span>    return new Triple&lt;&gt;(count.get(), mergedRegions, splitParents);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>  }<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span>  /**<a name="line.192"></a>
+<span class="sourceLineNo">193</span>   * If merged region no longer holds reference to the merge regions, archive<a name="line.193"></a>
+<span class="sourceLineNo">194</span>   * merge region on hdfs and perform deleting references in hbase:meta<a name="line.194"></a>
+<span class="sourceLineNo">195</span>   * @param mergedRegion<a name="line.195"></a>
+<span class="sourceLineNo">196</span>   * @return true if we delete references in merged region on hbase:meta and archive<a name="line.196"></a>
+<span class="sourceLineNo">197</span>   *         the files on the file system<a name="line.197"></a>
+<span class="sourceLineNo">198</span>   * @throws IOException<a name="line.198"></a>
+<span class="sourceLineNo">199</span>   */<a name="line.199"></a>
+<span class="sourceLineNo">200</span>  boolean cleanMergeRegion(final RegionInfo mergedRegion,<a name="line.200"></a>
+<span class="sourceLineNo">201</span>      final RegionInfo regionA, final RegionInfo regionB) throws IOException {<a name="line.201"></a>
+<span class="sourceLineNo">202</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.202"></a>
+<span class="sourceLineNo">203</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.203"></a>
+<span class="sourceLineNo">204</span>    Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());<a name="line.204"></a>
+<span class="sourceLineNo">205</span>    TableDescriptor htd = getTableDescriptor(mergedRegion.getTable());<a name="line.205"></a>
+<span class="sourceLineNo">206</span>    HRegionFileSystem regionFs = null;<a name="line.206"></a>
+<span class="sourceLineNo">207</span>    try {<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.208"></a>
+<span class="sourceLineNo">209</span>          this.services.getConfiguration(), fs, tabledir, mergedRegion, true);<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    } catch (IOException e) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    }<a name="line.212"></a>
+<span class="sourceLineNo">213</span>    if (regionFs == null || !regionFs.hasReferences(htd)) {<a name="line.213"></a>
+<span class="sourceLineNo">214</span>      LOG.debug("Deleting region " + regionA.getShortNameToLog() + " and "<a name="line.214"></a>
+<span class="sourceLineNo">215</span>          + regionB.getShortNameToLog()<a name="line.215"></a>
+<span class="sourceLineNo">216</span>          + " from fs because merged region no longer holds references");<a name="line.216"></a>
+<span class="sourceLineNo">217</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.217"></a>
+<span class="sourceLineNo">218</span>      pe.submitProcedure(new GCMergedRegionsProcedure(pe.getEnvironment(),<a name="line.218"></a>
+<span class="sourceLineNo">219</span>          mergedRegion, regionA, regionB));<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      // Remove from in-memory states<a name="line.220"></a>
+<span class="sourceLineNo">221</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionA);<a name="line.221"></a>
+<span class="sourceLineNo">222</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(regionB);<a name="line.222"></a>
+<span class="sourceLineNo">223</span>      this.services.getServerManager().removeRegion(regionA);<a name="line.223"></a>
+<span class="sourceLineNo">224</span>      this.services.getServerManager().removeRegion(regionB);<a name="line.224"></a>
+<span class="sourceLineNo">225</span>      return true;<a name="line.225"></a>
+<span class="sourceLineNo">226</span>    }<a name="line.226"></a>
+<span class="sourceLineNo">227</span>    return false;<a name="line.227"></a>
+<span class="sourceLineNo">228</span>  }<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>  /**<a name="line.230"></a>
+<span class="sourceLineNo">231</span>   * Run janitorial scan of catalog &lt;code&gt;hbase:meta&lt;/code&gt; table looking for<a name="line.231"></a>
+<span class="sourceLineNo">232</span>   * garbage to collect.<a name="line.232"></a>
+<span class="sourceLineNo">233</span>   * @return number of archiving jobs started.<a name="line.233"></a>
+<span class="sourceLineNo">234</span>   * @throws IOException<a name="line.234"></a>
+<span class="sourceLineNo">235</span>   */<a name="line.235"></a>
+<span class="sourceLineNo">236</span>  int scan() throws IOException {<a name="line.236"></a>
+<span class="sourceLineNo">237</span>    int result = 0;<a name="line.237"></a>
+<span class="sourceLineNo">238</span><a name="line.238"></a>
+<span class="sourceLineNo">239</span>    try {<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (!alreadyRunning.compareAndSet(false, true)) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        LOG.debug("CatalogJanitor already running");<a name="line.241"></a>
+<span class="sourceLineNo">242</span>        return result;<a name="line.242"></a>
+<span class="sourceLineNo">243</span>      }<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      Triple&lt;Integer, Map&lt;RegionInfo, Result&gt;, Map&lt;RegionInfo, Result&gt;&gt; scanTriple =<a name="line.244"></a>
+<span class="sourceLineNo">245</span>        getMergedRegionsAndSplitParents();<a name="line.245"></a>
+<span class="sourceLineNo">246</span>      /**<a name="line.246"></a>
+<span class="sourceLineNo">247</span>       * clean merge regions first<a name="line.247"></a>
+<span class="sourceLineNo">248</span>       */<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      Map&lt;RegionInfo, Result&gt; mergedRegions = scanTriple.getSecond();<a name="line.249"></a>
+<span class="sourceLineNo">250</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : mergedRegions.entrySet()) {<a name="line.250"></a>
+<span class="sourceLineNo">251</span>        if (this.services.isInMaintenanceMode()) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span>          // Stop cleaning if the master is in maintenance mode<a name="line.252"></a>
+<span class="sourceLineNo">253</span>          break;<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        }<a name="line.254"></a>
+<span class="sourceLineNo">255</span><a name="line.255"></a>
+<span class="sourceLineNo">256</span>        PairOfSameType&lt;RegionInfo&gt; p = MetaTableAccessor.getMergeRegions(e.getValue());<a name="line.256"></a>
+<span class="sourceLineNo">257</span>        RegionInfo regionA = p.getFirst();<a name="line.257"></a>
+<span class="sourceLineNo">258</span>        RegionInfo regionB = p.getSecond();<a name="line.258"></a>
+<span class="sourceLineNo">259</span>        if (regionA == null || regionB == null) {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>          LOG.warn("Unexpected references regionA="<a name="line.260"></a>
+<span class="sourceLineNo">261</span>              + (regionA == null ? "null" : regionA.getShortNameToLog())<a name="line.261"></a>
+<span class="sourceLineNo">262</span>              + ",regionB="<a name="line.262"></a>
+<span class="sourceLineNo">263</span>              + (regionB == null ? "null" : regionB.getShortNameToLog())<a name="line.263"></a>
+<span class="sourceLineNo">264</span>              + " in merged region " + e.getKey().getShortNameToLog());<a name="line.264"></a>
+<span class="sourceLineNo">265</span>        } else {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>          if (cleanMergeRegion(e.getKey(), regionA, regionB)) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span>            result++;<a name="line.267"></a>
+<span class="sourceLineNo">268</span>          }<a name="line.268"></a>
+<span class="sourceLineNo">269</span>        }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>      }<a name="line.270"></a>
+<span class="sourceLineNo">271</span>      /**<a name="line.271"></a>
+<span class="sourceLineNo">272</span>       * clean split parents<a name="line.272"></a>
+<span class="sourceLineNo">273</span>       */<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      Map&lt;RegionInfo, Result&gt; splitParents = scanTriple.getThird();<a name="line.274"></a>
+<span class="sourceLineNo">275</span><a name="line.275"></a>
+<span class="sourceLineNo">276</span>      // Now work on our list of found parents. See if any we can clean up.<a name="line.276"></a>
+<span class="sourceLineNo">277</span>      // regions whose parents are still around<a name="line.277"></a>
+<span class="sourceLineNo">278</span>      HashSet&lt;String&gt; parentNotCleaned = new HashSet&lt;&gt;();<a name="line.278"></a>
+<span class="sourceLineNo">279</span>      for (Map.Entry&lt;RegionInfo, Result&gt; e : splitParents.entrySet()) {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>        if (this.services.isInMaintenanceMode()) {<a name="line.280"></a>
+<span class="sourceLineNo">281</span>          // Stop cleaning if the master is in maintenance mode<a name="line.281"></a>
+<span class="sourceLineNo">282</span>          break;<a name="line.282"></a>
+<span class="sourceLineNo">283</span>        }<a name="line.283"></a>
+<span class="sourceLineNo">284</span><a name="line.284"></a>
+<span class="sourceLineNo">285</span>        if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &amp;&amp;<a name="line.285"></a>
+<span class="sourceLineNo">286</span>            cleanParent(e.getKey(), e.getValue())) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span>          result++;<a name="line.287"></a>
+<span class="sourceLineNo">288</span>        } else {<a name="line.288"></a>
+<span class="sourceLineNo">289</span>          // We could not clean the parent, so it's daughters should not be<a name="line.289"></a>
+<span class="sourceLineNo">290</span>          // cleaned either (HBASE-6160)<a name="line.290"></a>
+<span class="sourceLineNo">291</span>          PairOfSameType&lt;RegionInfo&gt; daughters =<a name="line.291"></a>
+<span class="sourceLineNo">292</span>              MetaTableAccessor.getDaughterRegions(e.getValue());<a name="line.292"></a>
+<span class="sourceLineNo">293</span>          parentNotCleaned.add(daughters.getFirst().getEncodedName());<a name="line.293"></a>
+<span class="sourceLineNo">294</span>          parentNotCleaned.add(daughters.getSecond().getEncodedName());<a name="line.294"></a>
+<span class="sourceLineNo">295</span>        }<a name="line.295"></a>
+<span class="sourceLineNo">296</span>      }<a name="line.296"></a>
+<span class="sourceLineNo">297</span>      return result;<a name="line.297"></a>
+<span class="sourceLineNo">298</span>    } finally {<a name="line.298"></a>
+<span class="sourceLineNo">299</span>      alreadyRunning.set(false);<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    }<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span><a name="line.302"></a>
+<span class="sourceLineNo">303</span>  /**<a name="line.303"></a>
+<span class="sourceLineNo">304</span>   * Compare HRegionInfos in a way that has split parents sort BEFORE their<a name="line.304"></a>
+<span class="sourceLineNo">305</span>   * daughters.<a name="line.305"></a>
+<span class="sourceLineNo">306</span>   */<a name="line.306"></a>
+<span class="sourceLineNo">307</span>  static class SplitParentFirstComparator implements Comparator&lt;RegionInfo&gt; {<a name="line.307"></a>
+<span class="sourceLineNo">308</span>    Comparator&lt;byte[]&gt; rowEndKeyComparator = new Bytes.RowEndKeyComparator();<a name="line.308"></a>
+<span class="sourceLineNo">309</span>    @Override<a name="line.309"></a>
+<span class="sourceLineNo">310</span>    public int compare(RegionInfo left, RegionInfo right) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span>      // This comparator differs from the one RegionInfo in that it sorts<a name="line.311"></a>
+<span class="sourceLineNo">312</span>      // parent before daughters.<a name="line.312"></a>
+<span class="sourceLineNo">313</span>      if (left == null) return -1;<a name="line.313"></a>
+<span class="sourceLineNo">314</span>      if (right == null) return 1;<a name="line.314"></a>
+<span class="sourceLineNo">315</span>      // Same table name.<a name="line.315"></a>
+<span class="sourceLineNo">316</span>      int result = left.getTable().compareTo(right.getTable());<a name="line.316"></a>
+<span class="sourceLineNo">317</span>      if (result != 0) return result;<a name="line.317"></a>
+<span class="sourceLineNo">318</span>      // Compare start keys.<a name="line.318"></a>
+<span class="sourceLineNo">319</span>      result = Bytes.compareTo(left.getStartKey(), right.getStartKey());<a name="line.319"></a>
+<span class="sourceLineNo">320</span>      if (result != 0) return result;<a name="line.320"></a>
+<span class="sourceLineNo">321</span>      // Compare end keys, but flip the operands so parent comes first<a name="line.321"></a>
+<span class="sourceLineNo">322</span>      result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());<a name="line.322"></a>
+<span class="sourceLineNo">323</span><a name="line.323"></a>
+<span class="sourceLineNo">324</span>      return result;<a name="line.324"></a>
+<span class="sourceLineNo">325</span>    }<a name="line.325"></a>
+<span class="sourceLineNo">326</span>  }<a name="line.326"></a>
+<span class="sourceLineNo">327</span><a name="line.327"></a>
+<span class="sourceLineNo">328</span>  /**<a name="line.328"></a>
+<span class="sourceLineNo">329</span>   * If daughters no longer hold reference to the parents, delete the parent.<a name="line.329"></a>
+<span class="sourceLineNo">330</span>   * @param parent RegionInfo of split offlined parent<a name="line.330"></a>
+<span class="sourceLineNo">331</span>   * @param rowContent Content of &lt;code&gt;parent&lt;/code&gt; row in<a name="line.331"></a>
+<span class="sourceLineNo">332</span>   * &lt;code&gt;metaRegionName&lt;/code&gt;<a name="line.332"></a>
+<span class="sourceLineNo">333</span>   * @return True if we removed &lt;code&gt;parent&lt;/code&gt; from meta table and from<a name="line.333"></a>
+<span class="sourceLineNo">334</span>   * the filesystem.<a name="line.334"></a>
+<span class="sourceLineNo">335</span>   * @throws IOException<a name="line.335"></a>
+<span class="sourceLineNo">336</span>   */<a name="line.336"></a>
+<span class="sourceLineNo">337</span>  boolean cleanParent(final RegionInfo parent, Result rowContent)<a name="line.337"></a>
+<span class="sourceLineNo">338</span>  throws IOException {<a name="line.338"></a>
+<span class="sourceLineNo">339</span>    // Check whether it is a merged region and not clean reference<a name="line.339"></a>
+<span class="sourceLineNo">340</span>    // No necessary to check MERGEB_QUALIFIER because these two qualifiers will<a name="line.340"></a>
+<span class="sourceLineNo">341</span>    // be inserted/deleted together<a name="line.341"></a>
+<span class="sourceLineNo">342</span>    if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {<a name="line.342"></a>
+<span class="sourceLineNo">343</span>      // wait cleaning merge region first<a name="line.343"></a>
+<span class="sourceLineNo">344</span>      return false;<a name="line.344"></a>
+<span class="sourceLineNo">345</span>    }<a name="line.345"></a>
+<span class="sourceLineNo">346</span>    // Run checks on each daughter split.<a name="line.346"></a>
+<span class="sourceLineNo">347</span>    PairOfSameType&lt;RegionInfo&gt; daughters = MetaTableAccessor.getDaughterRegions(rowContent);<a name="line.347"></a>
+<span class="sourceLineNo">348</span>    Pair&lt;Boolean, Boolean&gt; a = checkDaughterInFs(parent, daughters.getFirst());<a name="line.348"></a>
+<span class="sourceLineNo">349</span>    Pair&lt;Boolean, Boolean&gt; b = checkDaughterInFs(parent, daughters.getSecond());<a name="line.349"></a>
+<span class="sourceLineNo">350</span>    if (hasNoReferences(a) &amp;&amp; hasNoReferences(b)) {<a name="line.350"></a>
+<span class="sourceLineNo">351</span>      String daughterA = daughters.getFirst() != null?<a name="line.351"></a>
+<span class="sourceLineNo">352</span>          daughters.getFirst().getShortNameToLog(): "null";<a name="line.352"></a>
+<span class="sourceLineNo">353</span>      String daughterB = daughters.getSecond() != null?<a name="line.353"></a>
+<span class="sourceLineNo">354</span>          daughters.getSecond().getShortNameToLog(): "null";<a name="line.354"></a>
+<span class="sourceLineNo">355</span>      LOG.debug("Deleting region " + parent.getShortNameToLog() +<a name="line.355"></a>
+<span class="sourceLineNo">356</span>        " because daughters -- " + daughterA + ", " + daughterB +<a name="line.356"></a>
+<span class="sourceLineNo">357</span>        " -- no longer hold references");<a name="line.357"></a>
+<span class="sourceLineNo">358</span>      ProcedureExecutor&lt;MasterProcedureEnv&gt; pe = this.services.getMasterProcedureExecutor();<a name="line.358"></a>
+<span class="sourceLineNo">359</span>      pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));<a name="line.359"></a>
+<span class="sourceLineNo">360</span>      // Remove from in-memory states<a name="line.360"></a>
+<span class="sourceLineNo">361</span>      this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);<a name="line.361"></a>
+<span class="sourceLineNo">362</span>      this.services.getServerManager().removeRegion(parent);<a name="line.362"></a>
+<span class="sourceLineNo">363</span>      return true;<a name="line.363"></a>
+<span class="sourceLineNo">364</span>    }<a name="line.364"></a>
+<span class="sourceLineNo">365</span>    return false;<a name="line.365"></a>
+<span class="sourceLineNo">366</span>  }<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span>  /**<a name="line.368"></a>
+<span class="sourceLineNo">369</span>   * @param p A pair where the first boolean says whether or not the daughter<a name="line.369"></a>
+<span class="sourceLineNo">370</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.370"></a>
+<span class="sourceLineNo">371</span>   * whether the daughter has references to the parent.<a name="line.371"></a>
+<span class="sourceLineNo">372</span>   * @return True the passed &lt;code&gt;p&lt;/code&gt; signifies no references.<a name="line.372"></a>
+<span class="sourceLineNo">373</span>   */<a name="line.373"></a>
+<span class="sourceLineNo">374</span>  private boolean hasNoReferences(final Pair&lt;Boolean, Boolean&gt; p) {<a name="line.374"></a>
+<span class="sourceLineNo">375</span>    return !p.getFirst() || !p.getSecond();<a name="line.375"></a>
+<span class="sourceLineNo">376</span>  }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span>  /**<a name="line.378"></a>
+<span class="sourceLineNo">379</span>   * Checks if a daughter region -- either splitA or splitB -- still holds<a name="line.379"></a>
+<span class="sourceLineNo">380</span>   * references to parent.<a name="line.380"></a>
+<span class="sourceLineNo">381</span>   * @param parent Parent region<a name="line.381"></a>
+<span class="sourceLineNo">382</span>   * @param daughter Daughter region<a name="line.382"></a>
+<span class="sourceLineNo">383</span>   * @return A pair where the first boolean says whether or not the daughter<a name="line.383"></a>
+<span class="sourceLineNo">384</span>   * region directory exists in the filesystem and then the second boolean says<a name="line.384"></a>
+<span class="sourceLineNo">385</span>   * whether the daughter has references to the parent.<a name="line.385"></a>
+<span class="sourceLineNo">386</span>   * @throws IOException<a name="line.386"></a>
+<span class="sourceLineNo">387</span>   */<a name="line.387"></a>
+<span class="sourceLineNo">388</span>  Pair&lt;Boolean, Boolean&gt; checkDaughterInFs(final RegionInfo parent, final RegionInfo daughter)<a name="line.388"></a>
+<span class="sourceLineNo">389</span>  throws IOException {<a name="line.389"></a>
+<span class="sourceLineNo">390</span>    if (daughter == null)  {<a name="line.390"></a>
+<span class="sourceLineNo">391</span>      return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.391"></a>
+<span class="sourceLineNo">392</span>    }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span>    FileSystem fs = this.services.getMasterFileSystem().getFileSystem();<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    Path rootdir = this.services.getMasterFileSystem().getRootDir();<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());<a name="line.396"></a>
 <span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span>    HRegionFileSystem regionFs = null;<a name="line.398"></a>
+<span class="sourceLineNo">398</span>    Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());<a name="line.398"></a>
 <span class="sourceLineNo">399</span><a name="line.399"></a>
-<span class="sourceLineNo">400</span>    try {<a name="line.400"></a>
-<span class="sourceLineNo">401</span>      if (!FSUtils.isExists(fs, daughterRegionDir)) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span>        return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.402"></a>
-<span class="sourceLineNo">403</span>      }<a name="line.403"></a>
-<span class="sourceLineNo">404</span>    } catch (IOException ioe) {<a name="line.404"></a>
-<span class="sourceLineNo">405</span>      LOG.error("Error trying to determine if daughter region exists, " +<a name="line.405"></a>
-<span class="sourceLineNo">406</span>               "assuming exists and has references", ioe);<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.407"></a>
-<span class="sourceLineNo">408</span>    }<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>    boolean references = false;<a name="line.410"></a>
-<span class="sourceLineNo">411</span>    TableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());<a name="line.411"></a>
-<span class="sourceLineNo">412</span>    try {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.413"></a>
-<span class="sourceLineNo">414</span>          this.services.getConfiguration(), fs, tabledir, daughter, true);<a name="line.414"></a>
-<span class="sourceLineNo">415</span><a name="line.415"></a>
-<span class="sourceLineNo">416</span>      for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        if ((references = regionFs.hasReferences(family.getNameAsString()))) {<a name="line.417"></a>
-<span class="sourceLineNo">418</span>          break;<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        }<a name="line.419"></a>
-<span class="sourceLineNo">420</span>      }<a name="line.420"></a>
-<span class="sourceLineNo">421</span>    } catch (IOException e) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span>      LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()<a name="line.422"></a>
-<span class="sourceLineNo">423</span>          + ", to: " + parent.getEncodedName() + " assuming has references", e);<a name="line.423"></a>
-<span class="sourceLineNo">424</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.424"></a>
-<span class="sourceLineNo">425</span>    }<a name="line.425"></a>
-<span class="sourceLineNo">426</span>    return new Pair&lt;&gt;(Boolean.TRUE, Boolean.valueOf(references));<a name="line.426"></a>
-<span class="sourceLineNo">427</span>  }<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span>  private TableDescriptor getTableDescriptor(final TableName tableName)<a name="line.429"></a>
-<span class="sourceLineNo">430</span>      throws FileNotFoundException, IOException {<a name="line.430"></a>
-<span class="sourceLineNo">431</span>    return this.services.getTableDescriptors().get(tableName);<a name="line.431"></a>
-<span class="sourceLineNo">432</span>  }<a name="line.432"></a>
-<span class="sourceLineNo">433</span><a name="line.433"></a>
-<span class="sourceLineNo">434</span>  /**<a name="line.434"></a>
-<span class="sourceLineNo">435</span>   * Checks if the specified region has merge qualifiers, if so, try to clean<a name="line.435"></a>
-<span class="sourceLineNo">436</span>   * them<a name="line.436"></a>
-<span class="sourceLineNo">437</span>   * @param region<a name="line.437"></a>
-<span class="sourceLineNo">438</span>   * @return true if the specified region doesn't have merge qualifier now<a name="line.438"></a>
-<span class="sourceLineNo">439</span>   * @throws IOException<a name="line.439"></a>
-<span class="sourceLineNo">440</span>   */<a name="line.440"></a>
-<span class="sourceLineNo">441</span>  public boolean cleanMergeQualifier(final RegionInfo region)<a name="line.441"></a>
-<span class="sourceLineNo">442</span>      throws IOException {<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    // Get merge regions if it is a merged region and already has merge<a name="line.443"></a>
-<span class="sourceLineNo">444</span>    // qualifier<a name="line.444"></a>
-<span class="sourceLineNo">445</span>    Pair&lt;RegionInfo, RegionInfo&gt; mergeRegions = MetaTableAccessor<a name="line.445"></a>
-<span class="sourceLineNo">446</span>        .getRegionsFromMergeQualifier(this.services.getConnection(),<a name="line.446"></a>
-<span class="sourceLineNo">447</span>          region.getRegionName());<a name="line.447"></a>
-<span class="sourceLineNo">448</span>    if (mergeRegions == null<a name="line.448"></a>
-<span class="sourceLineNo">449</span>        || (mergeRegions.getFirst() == null &amp;&amp; mergeRegions.getSecond() == null)) {<a name="line.449"></a>
-<span class="sourceLineNo">450</span>      // It doesn't have merge qualifier, no need to clean<a name="line.450"></a>
-<span class="sourceLineNo">451</span>      return true;<a name="line.451"></a>
-<span class="sourceLineNo">452</span>    }<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    // It shouldn't happen, we must insert/delete these two qualifiers together<a name="line.453"></a>
-<span class="sourceLineNo">454</span>    if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {<a name="line.454"></a>
-<span class="sourceLineNo">455</span>      LOG.error("Merged region " + region.getRegionNameAsString()<a name="line.455"></a>
-<span class="sourceLineNo">456</span>          + " has only one merge qualifier in META.");<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      return false;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>    }<a name="line.458"></a>
-<span class="sourceLineNo">459</span>    return cleanMergeRegion(region, mergeRegions.getFirst(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        mergeRegions.getSecond());<a name="line.460"></a>
-<span class="sourceLineNo">461</span>  }<a name="line.461"></a>
-<span class="sourceLineNo">462</span>}<a name="line.462"></a>
+<span class="sourceLineNo">400</span>    HRegionFileSystem regionFs = null;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    try {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      if (!FSUtils.isExists(fs, daughterRegionDir)) {<a name="line.403"></a>
+<span class="sourceLineNo">404</span>        return new Pair&lt;&gt;(Boolean.FALSE, Boolean.FALSE);<a name="line.404"></a>
+<span class="sourceLineNo">405</span>      }<a name="line.405"></a>
+<span class="sourceLineNo">406</span>    } catch (IOException ioe) {<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      LOG.error("Error trying to determine if daughter region exists, " +<a name="line.407"></a>
+<span class="sourceLineNo">408</span>               "assuming exists and has references", ioe);<a name="line.408"></a>
+<span class="sourceLineNo">409</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>    }<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>    boolean references = false;<a name="line.412"></a>
+<span class="sourceLineNo">413</span>    TableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());<a name="line.413"></a>
+<span class="sourceLineNo">414</span>    try {<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      regionFs = HRegionFileSystem.openRegionFromFileSystem(<a name="line.415"></a>
+<span class="sourceLineNo">416</span>          this.services.getConfiguration(), fs, tabledir, daughter, true);<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>      for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        if ((references = regionFs.hasReferences(family.getNameAsString()))) {<a name="line.419"></a>
+<span class="sourceLineNo">420</span>          break;<a name="line.420"></a>
+<span class="sourceLineNo">421</span>        }<a name="line.421"></a>
+<span class="sourceLineNo">422</span>      }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>    } catch (IOException e) {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>      LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()<a name="line.424"></a>
+<span class="sourceLineNo">425</span>          + ", to: " + parent.getEncodedName() + " assuming has references", e);<a name="line.425"></a>
+<span class="sourceLineNo">426</span>      return new Pair&lt;&gt;(Boolean.TRUE, Boolean.TRUE);<a name="line.426"></a>
+<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
+<span class="sourceLineNo">428</span>    return new Pair&lt;&gt;(Boolean.TRUE, Boolean.valueOf(references));<a name="line.428"></a>
+<span class="sourceLineNo">429</span>  }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>  private TableDescriptor getTableDescriptor(final TableName tableName)<a name="line.431"></a>
+<span class="sourceLineNo">432</span>      throws FileNotFoundException, IOException {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>    return this.services.getTableDescriptors().get(tableName);<a name="line.433"></a>
+<span class="sourceLineNo">434</span>  }<a name="line.434"></a>
+<span class="sourceLineNo">435</span><a name="line.435"></a>
+<span class="sourceLineNo">436</span>  /**<a name="line.436"></a>
+<span class="sourceLineNo">437</span>   * Checks if the specified region has merge qualifiers, if so, try to clean<a name="line.437"></a>
+<span class="sourceLineNo">438</span>   * them<a name="line.438"></a>
+<span class="sourceLineNo">439</span>   * @param region<a name="line.439"></a>
+<span class="sourceLineNo">440</span>   * @return true if the specified region doesn't have merge qualifier now<a name="line.440"></a>
+<span class="sourceLineNo">441</span>   * @throws IOException<a name="line.441"></a>
+<span class="sourceLineNo">442</span>   */<a name="line.442"></a>
+<span class="sourceLineNo">443</span>  public boolean cleanMergeQualifier(final RegionInfo region)<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      throws IOException {<a name="line.444"></a>
+<span class="sourceLineNo">445</span>    // Get merge regions if it is a merged region and already has merge<a name="line.445"></a>
+<span class="sourceLineNo">446</span>    // qualifier<a name="line.446"></a>
+<span class="sourceLineNo">447</span>    Pair&lt;RegionInfo, RegionInfo&gt; mergeRegions = MetaTableAccessor<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        .getRegionsFromMergeQualifier(this.services.getConnection(),<a name="line.448"></a>
+<span class="sourceLineNo">449</span>          region.getRegionName());<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    if (mergeRegions == null<a name="line.450"></a>
+<span class="sourceLineNo">451</span>        || (mergeRegions.getFirst() == null &amp;&amp; mergeRegions.getSecond() == null)) {<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      // It doesn't have merge qualifier, no need to clean<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      return true;<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    }<a name="line.454"></a>
+<span class="sourceLineNo">455</span>    // It shouldn't happen, we must insert/delete these two qualifiers together<a name="line.455"></a>
+<span class="sourceLineNo">456</span>    if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {<a name="line.456"></a>
+<span class="sourceLineNo">457</span>      LOG.error("Merged region " + region.getRegionNameAsString()<a name="line.457"></a>
+<span class="sourceLineNo">458</span>          + " has only one merge qualifier in META.");<a name="line.458"></a>
+<span class="sourceLineNo">459</span>      return false;<a name="line.459"></a>
+<span class="sourceLineNo">460</span>    }<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    return cleanMergeRegion(region, mergeRegions.getFirst(),<a name="line.461"></a>
+<span class="sourceLineNo">462</span>        mergeRegions.getSecond());<a name="line.462"></a>
+<span class="sourceLineNo">463</span>  }<a name="line.463"></a>
+<span class="sourceLineNo">464</span>}<a name="line.464"></a>
 
 
 


[03/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
index 8ef0ff5..9545561 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.Iter.html
@@ -153,158 +153,161 @@
 <span class="sourceLineNo">145</span><a name="line.145"></a>
 <span class="sourceLineNo">146</span>  private void collectStackId(Entry entry, Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc,<a name="line.146"></a>
 <span class="sourceLineNo">147</span>      MutableInt maxStackId) {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      int stackId = entry.proc.getStackId(i);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>        maxStackId.setValue(stackId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    }<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.155"></a>
-<span class="sourceLineNo">156</span>  }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    for (Entry e : entry.subProcs) {<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    }<a name="line.164"></a>
-<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    for (Entry e : entry.subProcs) {<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.176"></a>
-<span class="sourceLineNo">177</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  // orphan procedures.<a name="line.178"></a>
-<span class="sourceLineNo">179</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    if (isFinished(rootEntry.proc)) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.183"></a>
-<span class="sourceLineNo">184</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.184"></a>
-<span class="sourceLineNo">185</span>      } else {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      }<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      return;<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.192"></a>
-<span class="sourceLineNo">193</span>    // the stack ids should start from 0 and increase by one every time<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    boolean valid = true;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      if (entries == null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          rootEntry);<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        valid = false;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      } else if (entries.size() &gt; 1) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.202"></a>
-<span class="sourceLineNo">203</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.203"></a>
-<span class="sourceLineNo">204</span>        valid = false;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      }<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    }<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    if (valid) {<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    } else {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
-<span class="sourceLineNo">213</span><a name="line.213"></a>
-<span class="sourceLineNo">214</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>    procMap.values().forEach(entry -&gt; {<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>    });<a name="line.218"></a>
-<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>  private static final class Iter implements ProcedureIterator {<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>    private ProtoAndProc current;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>      this.procs = procs;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      reset();<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    }<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span>    @Override<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    public void reset() {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      iter = procs.iterator();<a name="line.236"></a>
-<span class="sourceLineNo">237</span>      if (iter.hasNext()) {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        current = iter.next();<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      } else {<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        current = null;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>    }<a name="line.242"></a>
-<span class="sourceLineNo">243</span><a name="line.243"></a>
-<span class="sourceLineNo">244</span>    @Override<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    public boolean hasNext() {<a name="line.245"></a>
-<span class="sourceLineNo">246</span>      return current != null;<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    }<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span>    private void checkNext() {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      if (!hasNext()) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        throw new NoSuchElementException();<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span><a name="line.254"></a>
-<span class="sourceLineNo">255</span>    @Override<a name="line.255"></a>
-<span class="sourceLineNo">256</span>    public boolean isNextFinished() {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>      checkNext();<a name="line.257"></a>
-<span class="sourceLineNo">258</span>      return isFinished(current.proto);<a name="line.258"></a>
-<span class="sourceLineNo">259</span>    }<a name="line.259"></a>
-<span class="sourceLineNo">260</span><a name="line.260"></a>
-<span class="sourceLineNo">261</span>    private void moveToNext() {<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      if (iter.hasNext()) {<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        current = iter.next();<a name="line.263"></a>
-<span class="sourceLineNo">264</span>      } else {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>        current = null;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>    }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span>    @Override<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    public void skipNext() {<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      checkNext();<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      moveToNext();<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span>    @Override<a name="line.275"></a>
-<span class="sourceLineNo">276</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      checkNext();<a name="line.277"></a>
-<span class="sourceLineNo">278</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.278"></a>
-<span class="sourceLineNo">279</span>      moveToNext();<a name="line.279"></a>
-<span class="sourceLineNo">280</span>      return proc;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>    }<a name="line.281"></a>
-<span class="sourceLineNo">282</span>  }<a name="line.282"></a>
-<span class="sourceLineNo">283</span><a name="line.283"></a>
-<span class="sourceLineNo">284</span>  public ProcedureIterator getValidProcs() {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return new Iter(validProcs);<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    return new Iter(corruptedProcs);<a name="line.289"></a>
-<span class="sourceLineNo">290</span>  }<a name="line.290"></a>
-<span class="sourceLineNo">291</span><a name="line.291"></a>
-<span class="sourceLineNo">292</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    }<a name="line.296"></a>
-<span class="sourceLineNo">297</span>    return new WALProcedureTree(procMap);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>  }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>}<a name="line.299"></a>
+<span class="sourceLineNo">148</span>    if (LOG.isDebugEnabled()) {<a name="line.148"></a>
+<span class="sourceLineNo">149</span>      LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList());<a name="line.149"></a>
+<span class="sourceLineNo">150</span>    }<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      int stackId = entry.proc.getStackId(i);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>        maxStackId.setValue(stackId);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      }<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.158"></a>
+<span class="sourceLineNo">159</span>  }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    for (Entry e : entry.subProcs) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
+<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    for (Entry e : entry.subProcs) {<a name="line.173"></a>
+<span class="sourceLineNo">174</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
+<span class="sourceLineNo">176</span>  }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  // orphan procedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    if (isFinished(rootEntry.proc)) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.185"></a>
+<span class="sourceLineNo">186</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      } else {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>      }<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      return;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    }<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    // the stack ids should start from 0 and increase by one every time<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    boolean valid = true;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      if (entries == null) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.201"></a>
+<span class="sourceLineNo">202</span>          rootEntry);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        valid = false;<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      } else if (entries.size() &gt; 1) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.205"></a>
+<span class="sourceLineNo">206</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.206"></a>
+<span class="sourceLineNo">207</span>        valid = false;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      }<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    if (valid) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    } else {<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.213"></a>
+<span class="sourceLineNo">214</span>    }<a name="line.214"></a>
+<span class="sourceLineNo">215</span>  }<a name="line.215"></a>
+<span class="sourceLineNo">216</span><a name="line.216"></a>
+<span class="sourceLineNo">217</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    procMap.values().forEach(entry -&gt; {<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    });<a name="line.221"></a>
+<span class="sourceLineNo">222</span>  }<a name="line.222"></a>
+<span class="sourceLineNo">223</span><a name="line.223"></a>
+<span class="sourceLineNo">224</span>  private static final class Iter implements ProcedureIterator {<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.226"></a>
+<span class="sourceLineNo">227</span><a name="line.227"></a>
+<span class="sourceLineNo">228</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>    private ProtoAndProc current;<a name="line.230"></a>
+<span class="sourceLineNo">231</span><a name="line.231"></a>
+<span class="sourceLineNo">232</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      this.procs = procs;<a name="line.233"></a>
+<span class="sourceLineNo">234</span>      reset();<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    }<a name="line.235"></a>
+<span class="sourceLineNo">236</span><a name="line.236"></a>
+<span class="sourceLineNo">237</span>    @Override<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    public void reset() {<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      iter = procs.iterator();<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (iter.hasNext()) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        current = iter.next();<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      } else {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>        current = null;<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      }<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span><a name="line.246"></a>
+<span class="sourceLineNo">247</span>    @Override<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    public boolean hasNext() {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      return current != null;<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span>    private void checkNext() {<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      if (!hasNext()) {<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        throw new NoSuchElementException();<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      }<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    }<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span>    @Override<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    public boolean isNextFinished() {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      checkNext();<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      return isFinished(current.proto);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    private void moveToNext() {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      if (iter.hasNext()) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>        current = iter.next();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      } else {<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        current = null;<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>    }<a name="line.270"></a>
+<span class="sourceLineNo">271</span><a name="line.271"></a>
+<span class="sourceLineNo">272</span>    @Override<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    public void skipNext() {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      checkNext();<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      moveToNext();<a name="line.275"></a>
+<span class="sourceLineNo">276</span>    }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span>    @Override<a name="line.278"></a>
+<span class="sourceLineNo">279</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      checkNext();<a name="line.280"></a>
+<span class="sourceLineNo">281</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.281"></a>
+<span class="sourceLineNo">282</span>      moveToNext();<a name="line.282"></a>
+<span class="sourceLineNo">283</span>      return proc;<a name="line.283"></a>
+<span class="sourceLineNo">284</span>    }<a name="line.284"></a>
+<span class="sourceLineNo">285</span>  }<a name="line.285"></a>
+<span class="sourceLineNo">286</span><a name="line.286"></a>
+<span class="sourceLineNo">287</span>  public ProcedureIterator getValidProcs() {<a name="line.287"></a>
+<span class="sourceLineNo">288</span>    return new Iter(validProcs);<a name="line.288"></a>
+<span class="sourceLineNo">289</span>  }<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    return new Iter(corruptedProcs);<a name="line.292"></a>
+<span class="sourceLineNo">293</span>  }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.297"></a>
+<span class="sourceLineNo">298</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.298"></a>
+<span class="sourceLineNo">299</span>    }<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    return new WALProcedureTree(procMap);<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span>}<a name="line.302"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html
index 8ef0ff5..9545561 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.ProtoAndProc.html
@@ -153,158 +153,161 @@
 <span class="sourceLineNo">145</span><a name="line.145"></a>
 <span class="sourceLineNo">146</span>  private void collectStackId(Entry entry, Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc,<a name="line.146"></a>
 <span class="sourceLineNo">147</span>      MutableInt maxStackId) {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      int stackId = entry.proc.getStackId(i);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>        maxStackId.setValue(stackId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    }<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.155"></a>
-<span class="sourceLineNo">156</span>  }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    for (Entry e : entry.subProcs) {<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    }<a name="line.164"></a>
-<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    for (Entry e : entry.subProcs) {<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.176"></a>
-<span class="sourceLineNo">177</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  // orphan procedures.<a name="line.178"></a>
-<span class="sourceLineNo">179</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    if (isFinished(rootEntry.proc)) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.183"></a>
-<span class="sourceLineNo">184</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.184"></a>
-<span class="sourceLineNo">185</span>      } else {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      }<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      return;<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.192"></a>
-<span class="sourceLineNo">193</span>    // the stack ids should start from 0 and increase by one every time<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    boolean valid = true;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      if (entries == null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          rootEntry);<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        valid = false;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      } else if (entries.size() &gt; 1) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.202"></a>
-<span class="sourceLineNo">203</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.203"></a>
-<span class="sourceLineNo">204</span>        valid = false;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      }<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    }<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    if (valid) {<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    } else {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
-<span class="sourceLineNo">213</span><a name="line.213"></a>
-<span class="sourceLineNo">214</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>    procMap.values().forEach(entry -&gt; {<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>    });<a name="line.218"></a>
-<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>  private static final class Iter implements ProcedureIterator {<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>    private ProtoAndProc current;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>      this.procs = procs;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      reset();<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    }<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span>    @Override<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    public void reset() {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      iter = procs.iterator();<a name="line.236"></a>
-<span class="sourceLineNo">237</span>      if (iter.hasNext()) {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        current = iter.next();<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      } else {<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        current = null;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>    }<a name="line.242"></a>
-<span class="sourceLineNo">243</span><a name="line.243"></a>
-<span class="sourceLineNo">244</span>    @Override<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    public boolean hasNext() {<a name="line.245"></a>
-<span class="sourceLineNo">246</span>      return current != null;<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    }<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span>    private void checkNext() {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      if (!hasNext()) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        throw new NoSuchElementException();<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span><a name="line.254"></a>
-<span class="sourceLineNo">255</span>    @Override<a name="line.255"></a>
-<span class="sourceLineNo">256</span>    public boolean isNextFinished() {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>      checkNext();<a name="line.257"></a>
-<span class="sourceLineNo">258</span>      return isFinished(current.proto);<a name="line.258"></a>
-<span class="sourceLineNo">259</span>    }<a name="line.259"></a>
-<span class="sourceLineNo">260</span><a name="line.260"></a>
-<span class="sourceLineNo">261</span>    private void moveToNext() {<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      if (iter.hasNext()) {<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        current = iter.next();<a name="line.263"></a>
-<span class="sourceLineNo">264</span>      } else {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>        current = null;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>    }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span>    @Override<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    public void skipNext() {<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      checkNext();<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      moveToNext();<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span>    @Override<a name="line.275"></a>
-<span class="sourceLineNo">276</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      checkNext();<a name="line.277"></a>
-<span class="sourceLineNo">278</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.278"></a>
-<span class="sourceLineNo">279</span>      moveToNext();<a name="line.279"></a>
-<span class="sourceLineNo">280</span>      return proc;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>    }<a name="line.281"></a>
-<span class="sourceLineNo">282</span>  }<a name="line.282"></a>
-<span class="sourceLineNo">283</span><a name="line.283"></a>
-<span class="sourceLineNo">284</span>  public ProcedureIterator getValidProcs() {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return new Iter(validProcs);<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    return new Iter(corruptedProcs);<a name="line.289"></a>
-<span class="sourceLineNo">290</span>  }<a name="line.290"></a>
-<span class="sourceLineNo">291</span><a name="line.291"></a>
-<span class="sourceLineNo">292</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    }<a name="line.296"></a>
-<span class="sourceLineNo">297</span>    return new WALProcedureTree(procMap);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>  }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>}<a name="line.299"></a>
+<span class="sourceLineNo">148</span>    if (LOG.isDebugEnabled()) {<a name="line.148"></a>
+<span class="sourceLineNo">149</span>      LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList());<a name="line.149"></a>
+<span class="sourceLineNo">150</span>    }<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      int stackId = entry.proc.getStackId(i);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>        maxStackId.setValue(stackId);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      }<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.158"></a>
+<span class="sourceLineNo">159</span>  }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    for (Entry e : entry.subProcs) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
+<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    for (Entry e : entry.subProcs) {<a name="line.173"></a>
+<span class="sourceLineNo">174</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
+<span class="sourceLineNo">176</span>  }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  // orphan procedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    if (isFinished(rootEntry.proc)) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.185"></a>
+<span class="sourceLineNo">186</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      } else {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>      }<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      return;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    }<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    // the stack ids should start from 0 and increase by one every time<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    boolean valid = true;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      if (entries == null) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.201"></a>
+<span class="sourceLineNo">202</span>          rootEntry);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        valid = false;<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      } else if (entries.size() &gt; 1) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.205"></a>
+<span class="sourceLineNo">206</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.206"></a>
+<span class="sourceLineNo">207</span>        valid = false;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      }<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    if (valid) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    } else {<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.213"></a>
+<span class="sourceLineNo">214</span>    }<a name="line.214"></a>
+<span class="sourceLineNo">215</span>  }<a name="line.215"></a>
+<span class="sourceLineNo">216</span><a name="line.216"></a>
+<span class="sourceLineNo">217</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    procMap.values().forEach(entry -&gt; {<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    });<a name="line.221"></a>
+<span class="sourceLineNo">222</span>  }<a name="line.222"></a>
+<span class="sourceLineNo">223</span><a name="line.223"></a>
+<span class="sourceLineNo">224</span>  private static final class Iter implements ProcedureIterator {<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.226"></a>
+<span class="sourceLineNo">227</span><a name="line.227"></a>
+<span class="sourceLineNo">228</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>    private ProtoAndProc current;<a name="line.230"></a>
+<span class="sourceLineNo">231</span><a name="line.231"></a>
+<span class="sourceLineNo">232</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      this.procs = procs;<a name="line.233"></a>
+<span class="sourceLineNo">234</span>      reset();<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    }<a name="line.235"></a>
+<span class="sourceLineNo">236</span><a name="line.236"></a>
+<span class="sourceLineNo">237</span>    @Override<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    public void reset() {<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      iter = procs.iterator();<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (iter.hasNext()) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        current = iter.next();<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      } else {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>        current = null;<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      }<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span><a name="line.246"></a>
+<span class="sourceLineNo">247</span>    @Override<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    public boolean hasNext() {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      return current != null;<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span>    private void checkNext() {<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      if (!hasNext()) {<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        throw new NoSuchElementException();<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      }<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    }<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span>    @Override<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    public boolean isNextFinished() {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      checkNext();<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      return isFinished(current.proto);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    private void moveToNext() {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      if (iter.hasNext()) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>        current = iter.next();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      } else {<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        current = null;<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>    }<a name="line.270"></a>
+<span class="sourceLineNo">271</span><a name="line.271"></a>
+<span class="sourceLineNo">272</span>    @Override<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    public void skipNext() {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      checkNext();<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      moveToNext();<a name="line.275"></a>
+<span class="sourceLineNo">276</span>    }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span>    @Override<a name="line.278"></a>
+<span class="sourceLineNo">279</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      checkNext();<a name="line.280"></a>
+<span class="sourceLineNo">281</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.281"></a>
+<span class="sourceLineNo">282</span>      moveToNext();<a name="line.282"></a>
+<span class="sourceLineNo">283</span>      return proc;<a name="line.283"></a>
+<span class="sourceLineNo">284</span>    }<a name="line.284"></a>
+<span class="sourceLineNo">285</span>  }<a name="line.285"></a>
+<span class="sourceLineNo">286</span><a name="line.286"></a>
+<span class="sourceLineNo">287</span>  public ProcedureIterator getValidProcs() {<a name="line.287"></a>
+<span class="sourceLineNo">288</span>    return new Iter(validProcs);<a name="line.288"></a>
+<span class="sourceLineNo">289</span>  }<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    return new Iter(corruptedProcs);<a name="line.292"></a>
+<span class="sourceLineNo">293</span>  }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.297"></a>
+<span class="sourceLineNo">298</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.298"></a>
+<span class="sourceLineNo">299</span>    }<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    return new WALProcedureTree(procMap);<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span>}<a name="line.302"></a>
 
 
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
index 8ef0ff5..9545561 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureTree.html
@@ -153,158 +153,161 @@
 <span class="sourceLineNo">145</span><a name="line.145"></a>
 <span class="sourceLineNo">146</span>  private void collectStackId(Entry entry, Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc,<a name="line.146"></a>
 <span class="sourceLineNo">147</span>      MutableInt maxStackId) {<a name="line.147"></a>
-<span class="sourceLineNo">148</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.148"></a>
-<span class="sourceLineNo">149</span>      int stackId = entry.proc.getStackId(i);<a name="line.149"></a>
-<span class="sourceLineNo">150</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.150"></a>
-<span class="sourceLineNo">151</span>        maxStackId.setValue(stackId);<a name="line.151"></a>
-<span class="sourceLineNo">152</span>      }<a name="line.152"></a>
-<span class="sourceLineNo">153</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.153"></a>
-<span class="sourceLineNo">154</span>    }<a name="line.154"></a>
-<span class="sourceLineNo">155</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.155"></a>
-<span class="sourceLineNo">156</span>  }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.158"></a>
-<span class="sourceLineNo">159</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.159"></a>
-<span class="sourceLineNo">160</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.160"></a>
-<span class="sourceLineNo">161</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.161"></a>
-<span class="sourceLineNo">162</span>    for (Entry e : entry.subProcs) {<a name="line.162"></a>
-<span class="sourceLineNo">163</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.163"></a>
-<span class="sourceLineNo">164</span>    }<a name="line.164"></a>
-<span class="sourceLineNo">165</span>  }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.167"></a>
-<span class="sourceLineNo">168</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.168"></a>
-<span class="sourceLineNo">169</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.169"></a>
-<span class="sourceLineNo">170</span>    for (Entry e : entry.subProcs) {<a name="line.170"></a>
-<span class="sourceLineNo">171</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.171"></a>
-<span class="sourceLineNo">172</span>    }<a name="line.172"></a>
-<span class="sourceLineNo">173</span>  }<a name="line.173"></a>
-<span class="sourceLineNo">174</span><a name="line.174"></a>
-<span class="sourceLineNo">175</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.175"></a>
-<span class="sourceLineNo">176</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.176"></a>
-<span class="sourceLineNo">177</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.177"></a>
-<span class="sourceLineNo">178</span>  // orphan procedures.<a name="line.178"></a>
-<span class="sourceLineNo">179</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.179"></a>
-<span class="sourceLineNo">180</span>    if (isFinished(rootEntry.proc)) {<a name="line.180"></a>
-<span class="sourceLineNo">181</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.182"></a>
-<span class="sourceLineNo">183</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.183"></a>
-<span class="sourceLineNo">184</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.184"></a>
-<span class="sourceLineNo">185</span>      } else {<a name="line.185"></a>
-<span class="sourceLineNo">186</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.186"></a>
-<span class="sourceLineNo">187</span>      }<a name="line.187"></a>
-<span class="sourceLineNo">188</span>      return;<a name="line.188"></a>
-<span class="sourceLineNo">189</span>    }<a name="line.189"></a>
-<span class="sourceLineNo">190</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.190"></a>
-<span class="sourceLineNo">191</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.191"></a>
-<span class="sourceLineNo">192</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.192"></a>
-<span class="sourceLineNo">193</span>    // the stack ids should start from 0 and increase by one every time<a name="line.193"></a>
-<span class="sourceLineNo">194</span>    boolean valid = true;<a name="line.194"></a>
-<span class="sourceLineNo">195</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.196"></a>
-<span class="sourceLineNo">197</span>      if (entries == null) {<a name="line.197"></a>
-<span class="sourceLineNo">198</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.198"></a>
-<span class="sourceLineNo">199</span>          rootEntry);<a name="line.199"></a>
-<span class="sourceLineNo">200</span>        valid = false;<a name="line.200"></a>
-<span class="sourceLineNo">201</span>      } else if (entries.size() &gt; 1) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.202"></a>
-<span class="sourceLineNo">203</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.203"></a>
-<span class="sourceLineNo">204</span>        valid = false;<a name="line.204"></a>
-<span class="sourceLineNo">205</span>      }<a name="line.205"></a>
-<span class="sourceLineNo">206</span>    }<a name="line.206"></a>
-<span class="sourceLineNo">207</span>    if (valid) {<a name="line.207"></a>
-<span class="sourceLineNo">208</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.208"></a>
-<span class="sourceLineNo">209</span>    } else {<a name="line.209"></a>
-<span class="sourceLineNo">210</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.210"></a>
-<span class="sourceLineNo">211</span>    }<a name="line.211"></a>
-<span class="sourceLineNo">212</span>  }<a name="line.212"></a>
-<span class="sourceLineNo">213</span><a name="line.213"></a>
-<span class="sourceLineNo">214</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.214"></a>
-<span class="sourceLineNo">215</span>    procMap.values().forEach(entry -&gt; {<a name="line.215"></a>
-<span class="sourceLineNo">216</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.216"></a>
-<span class="sourceLineNo">217</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.217"></a>
-<span class="sourceLineNo">218</span>    });<a name="line.218"></a>
-<span class="sourceLineNo">219</span>  }<a name="line.219"></a>
-<span class="sourceLineNo">220</span><a name="line.220"></a>
-<span class="sourceLineNo">221</span>  private static final class Iter implements ProcedureIterator {<a name="line.221"></a>
-<span class="sourceLineNo">222</span><a name="line.222"></a>
-<span class="sourceLineNo">223</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.223"></a>
-<span class="sourceLineNo">224</span><a name="line.224"></a>
-<span class="sourceLineNo">225</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.225"></a>
-<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span>    private ProtoAndProc current;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.229"></a>
-<span class="sourceLineNo">230</span>      this.procs = procs;<a name="line.230"></a>
-<span class="sourceLineNo">231</span>      reset();<a name="line.231"></a>
-<span class="sourceLineNo">232</span>    }<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span>    @Override<a name="line.234"></a>
-<span class="sourceLineNo">235</span>    public void reset() {<a name="line.235"></a>
-<span class="sourceLineNo">236</span>      iter = procs.iterator();<a name="line.236"></a>
-<span class="sourceLineNo">237</span>      if (iter.hasNext()) {<a name="line.237"></a>
-<span class="sourceLineNo">238</span>        current = iter.next();<a name="line.238"></a>
-<span class="sourceLineNo">239</span>      } else {<a name="line.239"></a>
-<span class="sourceLineNo">240</span>        current = null;<a name="line.240"></a>
-<span class="sourceLineNo">241</span>      }<a name="line.241"></a>
-<span class="sourceLineNo">242</span>    }<a name="line.242"></a>
-<span class="sourceLineNo">243</span><a name="line.243"></a>
-<span class="sourceLineNo">244</span>    @Override<a name="line.244"></a>
-<span class="sourceLineNo">245</span>    public boolean hasNext() {<a name="line.245"></a>
-<span class="sourceLineNo">246</span>      return current != null;<a name="line.246"></a>
-<span class="sourceLineNo">247</span>    }<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span>    private void checkNext() {<a name="line.249"></a>
-<span class="sourceLineNo">250</span>      if (!hasNext()) {<a name="line.250"></a>
-<span class="sourceLineNo">251</span>        throw new NoSuchElementException();<a name="line.251"></a>
-<span class="sourceLineNo">252</span>      }<a name="line.252"></a>
-<span class="sourceLineNo">253</span>    }<a name="line.253"></a>
-<span class="sourceLineNo">254</span><a name="line.254"></a>
-<span class="sourceLineNo">255</span>    @Override<a name="line.255"></a>
-<span class="sourceLineNo">256</span>    public boolean isNextFinished() {<a name="line.256"></a>
-<span class="sourceLineNo">257</span>      checkNext();<a name="line.257"></a>
-<span class="sourceLineNo">258</span>      return isFinished(current.proto);<a name="line.258"></a>
-<span class="sourceLineNo">259</span>    }<a name="line.259"></a>
-<span class="sourceLineNo">260</span><a name="line.260"></a>
-<span class="sourceLineNo">261</span>    private void moveToNext() {<a name="line.261"></a>
-<span class="sourceLineNo">262</span>      if (iter.hasNext()) {<a name="line.262"></a>
-<span class="sourceLineNo">263</span>        current = iter.next();<a name="line.263"></a>
-<span class="sourceLineNo">264</span>      } else {<a name="line.264"></a>
-<span class="sourceLineNo">265</span>        current = null;<a name="line.265"></a>
-<span class="sourceLineNo">266</span>      }<a name="line.266"></a>
-<span class="sourceLineNo">267</span>    }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span>    @Override<a name="line.269"></a>
-<span class="sourceLineNo">270</span>    public void skipNext() {<a name="line.270"></a>
-<span class="sourceLineNo">271</span>      checkNext();<a name="line.271"></a>
-<span class="sourceLineNo">272</span>      moveToNext();<a name="line.272"></a>
-<span class="sourceLineNo">273</span>    }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span>    @Override<a name="line.275"></a>
-<span class="sourceLineNo">276</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.276"></a>
-<span class="sourceLineNo">277</span>      checkNext();<a name="line.277"></a>
-<span class="sourceLineNo">278</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.278"></a>
-<span class="sourceLineNo">279</span>      moveToNext();<a name="line.279"></a>
-<span class="sourceLineNo">280</span>      return proc;<a name="line.280"></a>
-<span class="sourceLineNo">281</span>    }<a name="line.281"></a>
-<span class="sourceLineNo">282</span>  }<a name="line.282"></a>
-<span class="sourceLineNo">283</span><a name="line.283"></a>
-<span class="sourceLineNo">284</span>  public ProcedureIterator getValidProcs() {<a name="line.284"></a>
-<span class="sourceLineNo">285</span>    return new Iter(validProcs);<a name="line.285"></a>
-<span class="sourceLineNo">286</span>  }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.288"></a>
-<span class="sourceLineNo">289</span>    return new Iter(corruptedProcs);<a name="line.289"></a>
-<span class="sourceLineNo">290</span>  }<a name="line.290"></a>
-<span class="sourceLineNo">291</span><a name="line.291"></a>
-<span class="sourceLineNo">292</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.292"></a>
-<span class="sourceLineNo">293</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.293"></a>
-<span class="sourceLineNo">294</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.294"></a>
-<span class="sourceLineNo">295</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.295"></a>
-<span class="sourceLineNo">296</span>    }<a name="line.296"></a>
-<span class="sourceLineNo">297</span>    return new WALProcedureTree(procMap);<a name="line.297"></a>
-<span class="sourceLineNo">298</span>  }<a name="line.298"></a>
-<span class="sourceLineNo">299</span>}<a name="line.299"></a>
+<span class="sourceLineNo">148</span>    if (LOG.isDebugEnabled()) {<a name="line.148"></a>
+<span class="sourceLineNo">149</span>      LOG.debug("Procedure {} stack ids={}", entry, entry.proc.getStackIdList());<a name="line.149"></a>
+<span class="sourceLineNo">150</span>    }<a name="line.150"></a>
+<span class="sourceLineNo">151</span>    for (int i = 0, n = entry.proc.getStackIdCount(); i &lt; n; i++) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span>      int stackId = entry.proc.getStackId(i);<a name="line.152"></a>
+<span class="sourceLineNo">153</span>      if (stackId &gt; maxStackId.intValue()) {<a name="line.153"></a>
+<span class="sourceLineNo">154</span>        maxStackId.setValue(stackId);<a name="line.154"></a>
+<span class="sourceLineNo">155</span>      }<a name="line.155"></a>
+<span class="sourceLineNo">156</span>      stackId2Proc.computeIfAbsent(stackId, k -&gt; new ArrayList&lt;&gt;()).add(entry);<a name="line.156"></a>
+<span class="sourceLineNo">157</span>    }<a name="line.157"></a>
+<span class="sourceLineNo">158</span>    entry.subProcs.forEach(e -&gt; collectStackId(e, stackId2Proc, maxStackId));<a name="line.158"></a>
+<span class="sourceLineNo">159</span>  }<a name="line.159"></a>
+<span class="sourceLineNo">160</span><a name="line.160"></a>
+<span class="sourceLineNo">161</span>  private void addAllToCorruptedAndRemoveFromProcMap(Entry entry,<a name="line.161"></a>
+<span class="sourceLineNo">162</span>      Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.162"></a>
+<span class="sourceLineNo">163</span>    corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.163"></a>
+<span class="sourceLineNo">164</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.164"></a>
+<span class="sourceLineNo">165</span>    for (Entry e : entry.subProcs) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span>      addAllToCorruptedAndRemoveFromProcMap(e, remainingProcMap);<a name="line.166"></a>
+<span class="sourceLineNo">167</span>    }<a name="line.167"></a>
+<span class="sourceLineNo">168</span>  }<a name="line.168"></a>
+<span class="sourceLineNo">169</span><a name="line.169"></a>
+<span class="sourceLineNo">170</span>  private void addAllToValidAndRemoveFromProcMap(Entry entry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span>    validProcs.add(new ProtoAndProc(entry.proc));<a name="line.171"></a>
+<span class="sourceLineNo">172</span>    remainingProcMap.remove(entry.proc.getProcId());<a name="line.172"></a>
+<span class="sourceLineNo">173</span>    for (Entry e : entry.subProcs) {<a name="line.173"></a>
+<span class="sourceLineNo">174</span>      addAllToValidAndRemoveFromProcMap(e, remainingProcMap);<a name="line.174"></a>
+<span class="sourceLineNo">175</span>    }<a name="line.175"></a>
+<span class="sourceLineNo">176</span>  }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span>  // In this method first we will check whether the given root procedure and all its sub procedures<a name="line.178"></a>
+<span class="sourceLineNo">179</span>  // are valid, through the procedure stack. And we will also remove all these procedures from the<a name="line.179"></a>
+<span class="sourceLineNo">180</span>  // remainingProcMap, so at last, if there are still procedures in the map, we know that there are<a name="line.180"></a>
+<span class="sourceLineNo">181</span>  // orphan procedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span>  private void checkReady(Entry rootEntry, Map&lt;Long, Entry&gt; remainingProcMap) {<a name="line.182"></a>
+<span class="sourceLineNo">183</span>    if (isFinished(rootEntry.proc)) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span>      if (!rootEntry.subProcs.isEmpty()) {<a name="line.184"></a>
+<span class="sourceLineNo">185</span>        LOG.error("unexpected active children for root-procedure: {}", rootEntry);<a name="line.185"></a>
+<span class="sourceLineNo">186</span>        rootEntry.subProcs.forEach(e -&gt; LOG.error("unexpected active children: {}", e));<a name="line.186"></a>
+<span class="sourceLineNo">187</span>        addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.187"></a>
+<span class="sourceLineNo">188</span>      } else {<a name="line.188"></a>
+<span class="sourceLineNo">189</span>        addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.189"></a>
+<span class="sourceLineNo">190</span>      }<a name="line.190"></a>
+<span class="sourceLineNo">191</span>      return;<a name="line.191"></a>
+<span class="sourceLineNo">192</span>    }<a name="line.192"></a>
+<span class="sourceLineNo">193</span>    Map&lt;Integer, List&lt;Entry&gt;&gt; stackId2Proc = new HashMap&lt;&gt;();<a name="line.193"></a>
+<span class="sourceLineNo">194</span>    MutableInt maxStackId = new MutableInt(Integer.MIN_VALUE);<a name="line.194"></a>
+<span class="sourceLineNo">195</span>    collectStackId(rootEntry, stackId2Proc, maxStackId);<a name="line.195"></a>
+<span class="sourceLineNo">196</span>    // the stack ids should start from 0 and increase by one every time<a name="line.196"></a>
+<span class="sourceLineNo">197</span>    boolean valid = true;<a name="line.197"></a>
+<span class="sourceLineNo">198</span>    for (int i = 0; i &lt;= maxStackId.intValue(); i++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span>      List&lt;Entry&gt; entries = stackId2Proc.get(i);<a name="line.199"></a>
+<span class="sourceLineNo">200</span>      if (entries == null) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span>        LOG.error("Missing stack id {}, max stack id is {}, root procedure is {}", i, maxStackId,<a name="line.201"></a>
+<span class="sourceLineNo">202</span>          rootEntry);<a name="line.202"></a>
+<span class="sourceLineNo">203</span>        valid = false;<a name="line.203"></a>
+<span class="sourceLineNo">204</span>      } else if (entries.size() &gt; 1) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span>        LOG.error("Multiple procedures {} have the same stack id {}, max stack id is {}," +<a name="line.205"></a>
+<span class="sourceLineNo">206</span>          " root procedure is {}", entries, i, maxStackId, rootEntry);<a name="line.206"></a>
+<span class="sourceLineNo">207</span>        valid = false;<a name="line.207"></a>
+<span class="sourceLineNo">208</span>      }<a name="line.208"></a>
+<span class="sourceLineNo">209</span>    }<a name="line.209"></a>
+<span class="sourceLineNo">210</span>    if (valid) {<a name="line.210"></a>
+<span class="sourceLineNo">211</span>      addAllToValidAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.211"></a>
+<span class="sourceLineNo">212</span>    } else {<a name="line.212"></a>
+<span class="sourceLineNo">213</span>      addAllToCorruptedAndRemoveFromProcMap(rootEntry, remainingProcMap);<a name="line.213"></a>
+<span class="sourceLineNo">214</span>    }<a name="line.214"></a>
+<span class="sourceLineNo">215</span>  }<a name="line.215"></a>
+<span class="sourceLineNo">216</span><a name="line.216"></a>
+<span class="sourceLineNo">217</span>  private void checkOrphan(Map&lt;Long, Entry&gt; procMap) {<a name="line.217"></a>
+<span class="sourceLineNo">218</span>    procMap.values().forEach(entry -&gt; {<a name="line.218"></a>
+<span class="sourceLineNo">219</span>      LOG.error("Orphan procedure: {}", entry);<a name="line.219"></a>
+<span class="sourceLineNo">220</span>      corruptedProcs.add(new ProtoAndProc(entry.proc));<a name="line.220"></a>
+<span class="sourceLineNo">221</span>    });<a name="line.221"></a>
+<span class="sourceLineNo">222</span>  }<a name="line.222"></a>
+<span class="sourceLineNo">223</span><a name="line.223"></a>
+<span class="sourceLineNo">224</span>  private static final class Iter implements ProcedureIterator {<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span>    private final List&lt;ProtoAndProc&gt; procs;<a name="line.226"></a>
+<span class="sourceLineNo">227</span><a name="line.227"></a>
+<span class="sourceLineNo">228</span>    private Iterator&lt;ProtoAndProc&gt; iter;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span>    private ProtoAndProc current;<a name="line.230"></a>
+<span class="sourceLineNo">231</span><a name="line.231"></a>
+<span class="sourceLineNo">232</span>    public Iter(List&lt;ProtoAndProc&gt; procs) {<a name="line.232"></a>
+<span class="sourceLineNo">233</span>      this.procs = procs;<a name="line.233"></a>
+<span class="sourceLineNo">234</span>      reset();<a name="line.234"></a>
+<span class="sourceLineNo">235</span>    }<a name="line.235"></a>
+<span class="sourceLineNo">236</span><a name="line.236"></a>
+<span class="sourceLineNo">237</span>    @Override<a name="line.237"></a>
+<span class="sourceLineNo">238</span>    public void reset() {<a name="line.238"></a>
+<span class="sourceLineNo">239</span>      iter = procs.iterator();<a name="line.239"></a>
+<span class="sourceLineNo">240</span>      if (iter.hasNext()) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span>        current = iter.next();<a name="line.241"></a>
+<span class="sourceLineNo">242</span>      } else {<a name="line.242"></a>
+<span class="sourceLineNo">243</span>        current = null;<a name="line.243"></a>
+<span class="sourceLineNo">244</span>      }<a name="line.244"></a>
+<span class="sourceLineNo">245</span>    }<a name="line.245"></a>
+<span class="sourceLineNo">246</span><a name="line.246"></a>
+<span class="sourceLineNo">247</span>    @Override<a name="line.247"></a>
+<span class="sourceLineNo">248</span>    public boolean hasNext() {<a name="line.248"></a>
+<span class="sourceLineNo">249</span>      return current != null;<a name="line.249"></a>
+<span class="sourceLineNo">250</span>    }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span>    private void checkNext() {<a name="line.252"></a>
+<span class="sourceLineNo">253</span>      if (!hasNext()) {<a name="line.253"></a>
+<span class="sourceLineNo">254</span>        throw new NoSuchElementException();<a name="line.254"></a>
+<span class="sourceLineNo">255</span>      }<a name="line.255"></a>
+<span class="sourceLineNo">256</span>    }<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span>    @Override<a name="line.258"></a>
+<span class="sourceLineNo">259</span>    public boolean isNextFinished() {<a name="line.259"></a>
+<span class="sourceLineNo">260</span>      checkNext();<a name="line.260"></a>
+<span class="sourceLineNo">261</span>      return isFinished(current.proto);<a name="line.261"></a>
+<span class="sourceLineNo">262</span>    }<a name="line.262"></a>
+<span class="sourceLineNo">263</span><a name="line.263"></a>
+<span class="sourceLineNo">264</span>    private void moveToNext() {<a name="line.264"></a>
+<span class="sourceLineNo">265</span>      if (iter.hasNext()) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span>        current = iter.next();<a name="line.266"></a>
+<span class="sourceLineNo">267</span>      } else {<a name="line.267"></a>
+<span class="sourceLineNo">268</span>        current = null;<a name="line.268"></a>
+<span class="sourceLineNo">269</span>      }<a name="line.269"></a>
+<span class="sourceLineNo">270</span>    }<a name="line.270"></a>
+<span class="sourceLineNo">271</span><a name="line.271"></a>
+<span class="sourceLineNo">272</span>    @Override<a name="line.272"></a>
+<span class="sourceLineNo">273</span>    public void skipNext() {<a name="line.273"></a>
+<span class="sourceLineNo">274</span>      checkNext();<a name="line.274"></a>
+<span class="sourceLineNo">275</span>      moveToNext();<a name="line.275"></a>
+<span class="sourceLineNo">276</span>    }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span>    @Override<a name="line.278"></a>
+<span class="sourceLineNo">279</span>    public Procedure&lt;?&gt; next() throws IOException {<a name="line.279"></a>
+<span class="sourceLineNo">280</span>      checkNext();<a name="line.280"></a>
+<span class="sourceLineNo">281</span>      Procedure&lt;?&gt; proc = current.getProc();<a name="line.281"></a>
+<span class="sourceLineNo">282</span>      moveToNext();<a name="line.282"></a>
+<span class="sourceLineNo">283</span>      return proc;<a name="line.283"></a>
+<span class="sourceLineNo">284</span>    }<a name="line.284"></a>
+<span class="sourceLineNo">285</span>  }<a name="line.285"></a>
+<span class="sourceLineNo">286</span><a name="line.286"></a>
+<span class="sourceLineNo">287</span>  public ProcedureIterator getValidProcs() {<a name="line.287"></a>
+<span class="sourceLineNo">288</span>    return new Iter(validProcs);<a name="line.288"></a>
+<span class="sourceLineNo">289</span>  }<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span>  public ProcedureIterator getCorruptedProcs() {<a name="line.291"></a>
+<span class="sourceLineNo">292</span>    return new Iter(corruptedProcs);<a name="line.292"></a>
+<span class="sourceLineNo">293</span>  }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span>  public static WALProcedureTree build(Collection&lt;ProcedureProtos.Procedure&gt; procedures) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span>    Map&lt;Long, Entry&gt; procMap = new HashMap&lt;&gt;();<a name="line.296"></a>
+<span class="sourceLineNo">297</span>    for (ProcedureProtos.Procedure proc : procedures) {<a name="line.297"></a>
+<span class="sourceLineNo">298</span>      procMap.put(proc.getProcId(), new Entry(proc));<a name="line.298"></a>
+<span class="sourceLineNo">299</span>    }<a name="line.299"></a>
+<span class="sourceLineNo">300</span>    return new WALProcedureTree(procMap);<a name="line.300"></a>
+<span class="sourceLineNo">301</span>  }<a name="line.301"></a>
+<span class="sourceLineNo">302</span>}<a name="line.302"></a>
 
 
 


[21/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.RegionReplicaCandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a

<TRUNCATED>

[13/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">835</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegi

<TRUNCATED>

[35/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
index c7d99b2..9d1542c 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.CandidateGenerator.html
@@ -382,1357 +382,1365 @@
 <span class="sourceLineNo">374</span>    for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.374"></a>
 <span class="sourceLineNo">375</span>      curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.375"></a>
 <span class="sourceLineNo">376</span>    }<a name="line.376"></a>
-<span class="sourceLineNo">377</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.377"></a>
-<span class="sourceLineNo">378</span>        + functionCost());<a name="line.378"></a>
+<span class="sourceLineNo">377</span>    double initCost = currentCost;<a name="line.377"></a>
+<span class="sourceLineNo">378</span>    double newCost = currentCost;<a name="line.378"></a>
 <span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span>    double initCost = currentCost;<a name="line.380"></a>
-<span class="sourceLineNo">381</span>    double newCost = currentCost;<a name="line.381"></a>
-<span class="sourceLineNo">382</span><a name="line.382"></a>
-<span class="sourceLineNo">383</span>    long computedMaxSteps;<a name="line.383"></a>
-<span class="sourceLineNo">384</span>    if (runMaxSteps) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.385"></a>
-<span class="sourceLineNo">386</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.386"></a>
-<span class="sourceLineNo">387</span>    } else {<a name="line.387"></a>
-<span class="sourceLineNo">388</span>      computedMaxSteps = Math.min(this.maxSteps,<a name="line.388"></a>
-<span class="sourceLineNo">389</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.389"></a>
-<span class="sourceLineNo">390</span>    }<a name="line.390"></a>
-<span class="sourceLineNo">391</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.391"></a>
-<span class="sourceLineNo">392</span>    long step;<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span>      Cluster.Action action = nextAction(cluster);<a name="line.395"></a>
-<span class="sourceLineNo">396</span><a name="line.396"></a>
-<span class="sourceLineNo">397</span>      if (action.type == Type.NULL) {<a name="line.397"></a>
-<span class="sourceLineNo">398</span>        continue;<a name="line.398"></a>
-<span class="sourceLineNo">399</span>      }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span>      cluster.doAction(action);<a name="line.401"></a>
-<span class="sourceLineNo">402</span>      updateCostsWithAction(cluster, action);<a name="line.402"></a>
-<span class="sourceLineNo">403</span><a name="line.403"></a>
-<span class="sourceLineNo">404</span>      newCost = computeCost(cluster, currentCost);<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span>      // Should this be kept?<a name="line.406"></a>
-<span class="sourceLineNo">407</span>      if (newCost &lt; currentCost) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span>        currentCost = newCost;<a name="line.408"></a>
-<span class="sourceLineNo">409</span><a name="line.409"></a>
-<span class="sourceLineNo">410</span>        // save for JMX<a name="line.410"></a>
-<span class="sourceLineNo">411</span>        curOverallCost = currentCost;<a name="line.411"></a>
-<span class="sourceLineNo">412</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.412"></a>
-<span class="sourceLineNo">413</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.413"></a>
-<span class="sourceLineNo">414</span>        }<a name="line.414"></a>
-<span class="sourceLineNo">415</span>      } else {<a name="line.415"></a>
-<span class="sourceLineNo">416</span>        // Put things back the way they were before.<a name="line.416"></a>
-<span class="sourceLineNo">417</span>        // TODO: undo by remembering old values<a name="line.417"></a>
-<span class="sourceLineNo">418</span>        Action undoAction = action.undoAction();<a name="line.418"></a>
-<span class="sourceLineNo">419</span>        cluster.doAction(undoAction);<a name="line.419"></a>
-<span class="sourceLineNo">420</span>        updateCostsWithAction(cluster, undoAction);<a name="line.420"></a>
-<span class="sourceLineNo">421</span>      }<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.423"></a>
-<span class="sourceLineNo">424</span>          maxRunningTime) {<a name="line.424"></a>
-<span class="sourceLineNo">425</span>        break;<a name="line.425"></a>
-<span class="sourceLineNo">426</span>      }<a name="line.426"></a>
-<span class="sourceLineNo">427</span>    }<a name="line.427"></a>
-<span class="sourceLineNo">428</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.428"></a>
-<span class="sourceLineNo">429</span><a name="line.429"></a>
-<span class="sourceLineNo">430</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span>    // update costs metrics<a name="line.432"></a>
-<span class="sourceLineNo">433</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.433"></a>
-<span class="sourceLineNo">434</span>    if (initCost &gt; currentCost) {<a name="line.434"></a>
-<span class="sourceLineNo">435</span>      plans = createRegionPlans(cluster);<a name="line.435"></a>
-<span class="sourceLineNo">436</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.436"></a>
-<span class="sourceLineNo">437</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.437"></a>
-<span class="sourceLineNo">438</span>        "{} regions; Going from a computed cost of {}" +<a name="line.438"></a>
-<span class="sourceLineNo">439</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.439"></a>
-<span class="sourceLineNo">440</span>        step, plans.size(), initCost, currentCost);<a name="line.440"></a>
-<span class="sourceLineNo">441</span>      return plans;<a name="line.441"></a>
-<span class="sourceLineNo">442</span>    }<a name="line.442"></a>
-<span class="sourceLineNo">443</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.443"></a>
-<span class="sourceLineNo">444</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.444"></a>
-<span class="sourceLineNo">445</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.445"></a>
-<span class="sourceLineNo">446</span>    return null;<a name="line.446"></a>
-<span class="sourceLineNo">447</span>  }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span>  /**<a name="line.449"></a>
-<span class="sourceLineNo">450</span>   * update costs to JMX<a name="line.450"></a>
-<span class="sourceLineNo">451</span>   */<a name="line.451"></a>
-<span class="sourceLineNo">452</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.452"></a>
-<span class="sourceLineNo">453</span>    if (tableName == null) return;<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.455"></a>
-<span class="sourceLineNo">456</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.456"></a>
-<span class="sourceLineNo">457</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.457"></a>
-<span class="sourceLineNo">458</span>      // overall cost<a name="line.458"></a>
-<span class="sourceLineNo">459</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.459"></a>
-<span class="sourceLineNo">460</span>        "Overall", "Overall cost", overall);<a name="line.460"></a>
-<span class="sourceLineNo">461</span><a name="line.461"></a>
-<span class="sourceLineNo">462</span>      // each cost function<a name="line.462"></a>
-<span class="sourceLineNo">463</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.463"></a>
-<span class="sourceLineNo">464</span>        CostFunction costFunction = costFunctions[i];<a name="line.464"></a>
-<span class="sourceLineNo">465</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.465"></a>
-<span class="sourceLineNo">466</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.466"></a>
-<span class="sourceLineNo">467</span>        // TODO: cost function may need a specific description<a name="line.467"></a>
-<span class="sourceLineNo">468</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.468"></a>
-<span class="sourceLineNo">469</span>          "The percent of " + costFunctionName, costPercent);<a name="line.469"></a>
-<span class="sourceLineNo">470</span>      }<a name="line.470"></a>
-<span class="sourceLineNo">471</span>    }<a name="line.471"></a>
-<span class="sourceLineNo">472</span>  }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span>  private String functionCost() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span>    StringBuilder builder = new StringBuilder();<a name="line.475"></a>
-<span class="sourceLineNo">476</span>    for (CostFunction c:costFunctions) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span>      builder.append(c.getClass().getSimpleName());<a name="line.477"></a>
-<span class="sourceLineNo">478</span>      builder.append(" : (");<a name="line.478"></a>
-<span class="sourceLineNo">479</span>      builder.append(c.getMultiplier());<a name="line.479"></a>
-<span class="sourceLineNo">480</span>      builder.append(", ");<a name="line.480"></a>
-<span class="sourceLineNo">481</span>      builder.append(c.cost());<a name="line.481"></a>
-<span class="sourceLineNo">482</span>      builder.append("); ");<a name="line.482"></a>
-<span class="sourceLineNo">483</span>    }<a name="line.483"></a>
-<span class="sourceLineNo">484</span>    return builder.toString();<a name="line.484"></a>
-<span class="sourceLineNo">485</span>  }<a name="line.485"></a>
-<span class="sourceLineNo">486</span><a name="line.486"></a>
-<span class="sourceLineNo">487</span>  /**<a name="line.487"></a>
-<span class="sourceLineNo">488</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.488"></a>
-<span class="sourceLineNo">489</span>   * state.<a name="line.489"></a>
-<span class="sourceLineNo">490</span>   *<a name="line.490"></a>
-<span class="sourceLineNo">491</span>   * @param cluster The state of the cluster<a name="line.491"></a>
-<span class="sourceLineNo">492</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.492"></a>
-<span class="sourceLineNo">493</span>   */<a name="line.493"></a>
-<span class="sourceLineNo">494</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.494"></a>
-<span class="sourceLineNo">495</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.495"></a>
-<span class="sourceLineNo">496</span>    for (int regionIndex = 0;<a name="line.496"></a>
-<span class="sourceLineNo">497</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.497"></a>
-<span class="sourceLineNo">498</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.498"></a>
-<span class="sourceLineNo">499</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.499"></a>
-<span class="sourceLineNo">500</span><a name="line.500"></a>
-<span class="sourceLineNo">501</span>      if (initialServerIndex != newServerIndex) {<a name="line.501"></a>
-<span class="sourceLineNo">502</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.502"></a>
-<span class="sourceLineNo">503</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.503"></a>
-<span class="sourceLineNo">504</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span>        if (LOG.isTraceEnabled()) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.507"></a>
-<span class="sourceLineNo">508</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.508"></a>
-<span class="sourceLineNo">509</span>        }<a name="line.509"></a>
-<span class="sourceLineNo">510</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.510"></a>
-<span class="sourceLineNo">511</span>        plans.add(rp);<a name="line.511"></a>
-<span class="sourceLineNo">512</span>      }<a name="line.512"></a>
-<span class="sourceLineNo">513</span>    }<a name="line.513"></a>
-<span class="sourceLineNo">514</span>    return plans;<a name="line.514"></a>
-<span class="sourceLineNo">515</span>  }<a name="line.515"></a>
-<span class="sourceLineNo">516</span><a name="line.516"></a>
-<span class="sourceLineNo">517</span>  /**<a name="line.517"></a>
-<span class="sourceLineNo">518</span>   * Store the current region loads.<a name="line.518"></a>
-<span class="sourceLineNo">519</span>   */<a name="line.519"></a>
-<span class="sourceLineNo">520</span>  private synchronized void updateRegionLoad() {<a name="line.520"></a>
-<span class="sourceLineNo">521</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.521"></a>
-<span class="sourceLineNo">522</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.522"></a>
-<span class="sourceLineNo">523</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.523"></a>
-<span class="sourceLineNo">524</span>    loads = new HashMap&lt;&gt;();<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.526"></a>
-<span class="sourceLineNo">527</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.527"></a>
-<span class="sourceLineNo">528</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.528"></a>
-<span class="sourceLineNo">529</span>        if (rLoads == null) {<a name="line.529"></a>
-<span class="sourceLineNo">530</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.530"></a>
-<span class="sourceLineNo">531</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.531"></a>
-<span class="sourceLineNo">532</span>          rLoads.remove();<a name="line.532"></a>
-<span class="sourceLineNo">533</span>        }<a name="line.533"></a>
-<span class="sourceLineNo">534</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.534"></a>
-<span class="sourceLineNo">535</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.535"></a>
-<span class="sourceLineNo">536</span>      });<a name="line.536"></a>
-<span class="sourceLineNo">537</span>    });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.539"></a>
-<span class="sourceLineNo">540</span>      cost.setLoads(loads);<a name="line.540"></a>
-<span class="sourceLineNo">541</span>    }<a name="line.541"></a>
-<span class="sourceLineNo">542</span>  }<a name="line.542"></a>
-<span class="sourceLineNo">543</span><a name="line.543"></a>
-<span class="sourceLineNo">544</span>  protected void initCosts(Cluster cluster) {<a name="line.544"></a>
-<span class="sourceLineNo">545</span>    for (CostFunction c:costFunctions) {<a name="line.545"></a>
-<span class="sourceLineNo">546</span>      c.init(cluster);<a name="line.546"></a>
-<span class="sourceLineNo">547</span>    }<a name="line.547"></a>
-<span class="sourceLineNo">548</span>  }<a name="line.548"></a>
-<span class="sourceLineNo">549</span><a name="line.549"></a>
-<span class="sourceLineNo">550</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.550"></a>
-<span class="sourceLineNo">551</span>    for (CostFunction c : costFunctions) {<a name="line.551"></a>
-<span class="sourceLineNo">552</span>      c.postAction(action);<a name="line.552"></a>
-<span class="sourceLineNo">553</span>    }<a name="line.553"></a>
-<span class="sourceLineNo">554</span>  }<a name="line.554"></a>
-<span class="sourceLineNo">555</span><a name="line.555"></a>
-<span class="sourceLineNo">556</span>  /**<a name="line.556"></a>
-<span class="sourceLineNo">557</span>   * Get the names of the cost functions<a name="line.557"></a>
-<span class="sourceLineNo">558</span>   */<a name="line.558"></a>
-<span class="sourceLineNo">559</span>  public String[] getCostFunctionNames() {<a name="line.559"></a>
-<span class="sourceLineNo">560</span>    if (costFunctions == null) return null;<a name="line.560"></a>
-<span class="sourceLineNo">561</span>    String[] ret = new String[costFunctions.length];<a name="line.561"></a>
-<span class="sourceLineNo">562</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.562"></a>
-<span class="sourceLineNo">563</span>      CostFunction c = costFunctions[i];<a name="line.563"></a>
-<span class="sourceLineNo">564</span>      ret[i] = c.getClass().getSimpleName();<a name="line.564"></a>
-<span class="sourceLineNo">565</span>    }<a name="line.565"></a>
-<span class="sourceLineNo">566</span><a name="line.566"></a>
-<span class="sourceLineNo">567</span>    return ret;<a name="line.567"></a>
-<span class="sourceLineNo">568</span>  }<a name="line.568"></a>
-<span class="sourceLineNo">569</span><a name="line.569"></a>
-<span class="sourceLineNo">570</span>  /**<a name="line.570"></a>
-<span class="sourceLineNo">571</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.571"></a>
-<span class="sourceLineNo">572</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.572"></a>
-<span class="sourceLineNo">573</span>   *<a name="line.573"></a>
-<span class="sourceLineNo">574</span>   * @param cluster The state of the cluster<a name="line.574"></a>
-<span class="sourceLineNo">575</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.575"></a>
-<span class="sourceLineNo">576</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.576"></a>
-<span class="sourceLineNo">577</span>   *         aggregate of all individual cost functions.<a name="line.577"></a>
-<span class="sourceLineNo">578</span>   */<a name="line.578"></a>
-<span class="sourceLineNo">579</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.579"></a>
-<span class="sourceLineNo">580</span>    double total = 0;<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.582"></a>
-<span class="sourceLineNo">583</span>      CostFunction c = costFunctions[i];<a name="line.583"></a>
-<span class="sourceLineNo">584</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.584"></a>
-<span class="sourceLineNo">585</span><a name="line.585"></a>
-<span class="sourceLineNo">586</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.586"></a>
-<span class="sourceLineNo">587</span>        continue;<a name="line.587"></a>
-<span class="sourceLineNo">588</span>      }<a name="line.588"></a>
+<span class="sourceLineNo">380</span>    long computedMaxSteps;<a name="line.380"></a>
+<span class="sourceLineNo">381</span>    if (runMaxSteps) {<a name="line.381"></a>
+<span class="sourceLineNo">382</span>      computedMaxSteps = Math.max(this.maxSteps,<a name="line.382"></a>
+<span class="sourceLineNo">383</span>          ((long)cluster.numRegions * (long)this.stepsPerRegion * (long)cluster.numServers));<a name="line.383"></a>
+<span class="sourceLineNo">384</span>    } else {<a name="line.384"></a>
+<span class="sourceLineNo">385</span>      long calculatedMaxSteps = (long)cluster.numRegions * (long)this.stepsPerRegion *<a name="line.385"></a>
+<span class="sourceLineNo">386</span>          (long)cluster.numServers;<a name="line.386"></a>
+<span class="sourceLineNo">387</span>      computedMaxSteps = Math.min(this.maxSteps, calculatedMaxSteps);<a name="line.387"></a>
+<span class="sourceLineNo">388</span>      if (calculatedMaxSteps &gt; maxSteps) {<a name="line.388"></a>
+<span class="sourceLineNo">389</span>        LOG.warn("calculatedMaxSteps:{} for loadbalancer's stochastic walk is larger than "<a name="line.389"></a>
+<span class="sourceLineNo">390</span>            + "maxSteps:{}. Hence load balancing may not work well. Setting parameter "<a name="line.390"></a>
+<span class="sourceLineNo">391</span>            + "\"hbase.master.balancer.stochastic.runMaxSteps\" to true can overcome this issue."<a name="line.391"></a>
+<span class="sourceLineNo">392</span>            + "(This config change does not require service restart)", calculatedMaxSteps,<a name="line.392"></a>
+<span class="sourceLineNo">393</span>            maxRunningTime);<a name="line.393"></a>
+<span class="sourceLineNo">394</span>      }<a name="line.394"></a>
+<span class="sourceLineNo">395</span>    }<a name="line.395"></a>
+<span class="sourceLineNo">396</span>    LOG.info("start StochasticLoadBalancer.balancer, initCost=" + currentCost + ", functionCost="<a name="line.396"></a>
+<span class="sourceLineNo">397</span>        + functionCost() + " computedMaxSteps: " + computedMaxSteps);<a name="line.397"></a>
+<span class="sourceLineNo">398</span><a name="line.398"></a>
+<span class="sourceLineNo">399</span>    // Perform a stochastic walk to see if we can get a good fit.<a name="line.399"></a>
+<span class="sourceLineNo">400</span>    long step;<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span>    for (step = 0; step &lt; computedMaxSteps; step++) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span>      Cluster.Action action = nextAction(cluster);<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span>      if (action.type == Type.NULL) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span>        continue;<a name="line.406"></a>
+<span class="sourceLineNo">407</span>      }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span>      cluster.doAction(action);<a name="line.409"></a>
+<span class="sourceLineNo">410</span>      updateCostsWithAction(cluster, action);<a name="line.410"></a>
+<span class="sourceLineNo">411</span><a name="line.411"></a>
+<span class="sourceLineNo">412</span>      newCost = computeCost(cluster, currentCost);<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span>      // Should this be kept?<a name="line.414"></a>
+<span class="sourceLineNo">415</span>      if (newCost &lt; currentCost) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span>        currentCost = newCost;<a name="line.416"></a>
+<span class="sourceLineNo">417</span><a name="line.417"></a>
+<span class="sourceLineNo">418</span>        // save for JMX<a name="line.418"></a>
+<span class="sourceLineNo">419</span>        curOverallCost = currentCost;<a name="line.419"></a>
+<span class="sourceLineNo">420</span>        for (int i = 0; i &lt; this.curFunctionCosts.length; i++) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span>          curFunctionCosts[i] = tempFunctionCosts[i];<a name="line.421"></a>
+<span class="sourceLineNo">422</span>        }<a name="line.422"></a>
+<span class="sourceLineNo">423</span>      } else {<a name="line.423"></a>
+<span class="sourceLineNo">424</span>        // Put things back the way they were before.<a name="line.424"></a>
+<span class="sourceLineNo">425</span>        // TODO: undo by remembering old values<a name="line.425"></a>
+<span class="sourceLineNo">426</span>        Action undoAction = action.undoAction();<a name="line.426"></a>
+<span class="sourceLineNo">427</span>        cluster.doAction(undoAction);<a name="line.427"></a>
+<span class="sourceLineNo">428</span>        updateCostsWithAction(cluster, undoAction);<a name="line.428"></a>
+<span class="sourceLineNo">429</span>      }<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span>      if (EnvironmentEdgeManager.currentTime() - startTime &gt;<a name="line.431"></a>
+<span class="sourceLineNo">432</span>          maxRunningTime) {<a name="line.432"></a>
+<span class="sourceLineNo">433</span>        break;<a name="line.433"></a>
+<span class="sourceLineNo">434</span>      }<a name="line.434"></a>
+<span class="sourceLineNo">435</span>    }<a name="line.435"></a>
+<span class="sourceLineNo">436</span>    long endTime = EnvironmentEdgeManager.currentTime();<a name="line.436"></a>
+<span class="sourceLineNo">437</span><a name="line.437"></a>
+<span class="sourceLineNo">438</span>    metricsBalancer.balanceCluster(endTime - startTime);<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span>    // update costs metrics<a name="line.440"></a>
+<span class="sourceLineNo">441</span>    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);<a name="line.441"></a>
+<span class="sourceLineNo">442</span>    if (initCost &gt; currentCost) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span>      plans = createRegionPlans(cluster);<a name="line.443"></a>
+<span class="sourceLineNo">444</span>      LOG.info("Finished computing new load balance plan. Computation took {}" +<a name="line.444"></a>
+<span class="sourceLineNo">445</span>        " to try {} different iterations.  Found a solution that moves " +<a name="line.445"></a>
+<span class="sourceLineNo">446</span>        "{} regions; Going from a computed cost of {}" +<a name="line.446"></a>
+<span class="sourceLineNo">447</span>        " to a new cost of {}", java.time.Duration.ofMillis(endTime - startTime),<a name="line.447"></a>
+<span class="sourceLineNo">448</span>        step, plans.size(), initCost, currentCost);<a name="line.448"></a>
+<span class="sourceLineNo">449</span>      return plans;<a name="line.449"></a>
+<span class="sourceLineNo">450</span>    }<a name="line.450"></a>
+<span class="sourceLineNo">451</span>    LOG.info("Could not find a better load balance plan.  Tried {} different configurations in " +<a name="line.451"></a>
+<span class="sourceLineNo">452</span>      "{}, and did not find anything with a computed cost less than {}", step,<a name="line.452"></a>
+<span class="sourceLineNo">453</span>      java.time.Duration.ofMillis(endTime - startTime), initCost);<a name="line.453"></a>
+<span class="sourceLineNo">454</span>    return null;<a name="line.454"></a>
+<span class="sourceLineNo">455</span>  }<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span>  /**<a name="line.457"></a>
+<span class="sourceLineNo">458</span>   * update costs to JMX<a name="line.458"></a>
+<span class="sourceLineNo">459</span>   */<a name="line.459"></a>
+<span class="sourceLineNo">460</span>  private void updateStochasticCosts(TableName tableName, Double overall, Double[] subCosts) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span>    if (tableName == null) return;<a name="line.461"></a>
+<span class="sourceLineNo">462</span><a name="line.462"></a>
+<span class="sourceLineNo">463</span>    // check if the metricsBalancer is MetricsStochasticBalancer before casting<a name="line.463"></a>
+<span class="sourceLineNo">464</span>    if (metricsBalancer instanceof MetricsStochasticBalancer) {<a name="line.464"></a>
+<span class="sourceLineNo">465</span>      MetricsStochasticBalancer balancer = (MetricsStochasticBalancer) metricsBalancer;<a name="line.465"></a>
+<span class="sourceLineNo">466</span>      // overall cost<a name="line.466"></a>
+<span class="sourceLineNo">467</span>      balancer.updateStochasticCost(tableName.getNameAsString(),<a name="line.467"></a>
+<span class="sourceLineNo">468</span>        "Overall", "Overall cost", overall);<a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span>      // each cost function<a name="line.470"></a>
+<span class="sourceLineNo">471</span>      for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.471"></a>
+<span class="sourceLineNo">472</span>        CostFunction costFunction = costFunctions[i];<a name="line.472"></a>
+<span class="sourceLineNo">473</span>        String costFunctionName = costFunction.getClass().getSimpleName();<a name="line.473"></a>
+<span class="sourceLineNo">474</span>        Double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);<a name="line.474"></a>
+<span class="sourceLineNo">475</span>        // TODO: cost function may need a specific description<a name="line.475"></a>
+<span class="sourceLineNo">476</span>        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,<a name="line.476"></a>
+<span class="sourceLineNo">477</span>          "The percent of " + costFunctionName, costPercent);<a name="line.477"></a>
+<span class="sourceLineNo">478</span>      }<a name="line.478"></a>
+<span class="sourceLineNo">479</span>    }<a name="line.479"></a>
+<span class="sourceLineNo">480</span>  }<a name="line.480"></a>
+<span class="sourceLineNo">481</span><a name="line.481"></a>
+<span class="sourceLineNo">482</span>  private String functionCost() {<a name="line.482"></a>
+<span class="sourceLineNo">483</span>    StringBuilder builder = new StringBuilder();<a name="line.483"></a>
+<span class="sourceLineNo">484</span>    for (CostFunction c:costFunctions) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span>      builder.append(c.getClass().getSimpleName());<a name="line.485"></a>
+<span class="sourceLineNo">486</span>      builder.append(" : (");<a name="line.486"></a>
+<span class="sourceLineNo">487</span>      builder.append(c.getMultiplier());<a name="line.487"></a>
+<span class="sourceLineNo">488</span>      builder.append(", ");<a name="line.488"></a>
+<span class="sourceLineNo">489</span>      builder.append(c.cost());<a name="line.489"></a>
+<span class="sourceLineNo">490</span>      builder.append("); ");<a name="line.490"></a>
+<span class="sourceLineNo">491</span>    }<a name="line.491"></a>
+<span class="sourceLineNo">492</span>    return builder.toString();<a name="line.492"></a>
+<span class="sourceLineNo">493</span>  }<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span>  /**<a name="line.495"></a>
+<span class="sourceLineNo">496</span>   * Create all of the RegionPlan's needed to move from the initial cluster state to the desired<a name="line.496"></a>
+<span class="sourceLineNo">497</span>   * state.<a name="line.497"></a>
+<span class="sourceLineNo">498</span>   *<a name="line.498"></a>
+<span class="sourceLineNo">499</span>   * @param cluster The state of the cluster<a name="line.499"></a>
+<span class="sourceLineNo">500</span>   * @return List of RegionPlan's that represent the moves needed to get to desired final state.<a name="line.500"></a>
+<span class="sourceLineNo">501</span>   */<a name="line.501"></a>
+<span class="sourceLineNo">502</span>  private List&lt;RegionPlan&gt; createRegionPlans(Cluster cluster) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span>    List&lt;RegionPlan&gt; plans = new LinkedList&lt;&gt;();<a name="line.503"></a>
+<span class="sourceLineNo">504</span>    for (int regionIndex = 0;<a name="line.504"></a>
+<span class="sourceLineNo">505</span>         regionIndex &lt; cluster.regionIndexToServerIndex.length; regionIndex++) {<a name="line.505"></a>
+<span class="sourceLineNo">506</span>      int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];<a name="line.506"></a>
+<span class="sourceLineNo">507</span>      int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span>      if (initialServerIndex != newServerIndex) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span>        RegionInfo region = cluster.regions[regionIndex];<a name="line.510"></a>
+<span class="sourceLineNo">511</span>        ServerName initialServer = cluster.servers[initialServerIndex];<a name="line.511"></a>
+<span class="sourceLineNo">512</span>        ServerName newServer = cluster.servers[newServerIndex];<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span>        if (LOG.isTraceEnabled()) {<a name="line.514"></a>
+<span class="sourceLineNo">515</span>          LOG.trace("Moving Region " + region.getEncodedName() + " from server "<a name="line.515"></a>
+<span class="sourceLineNo">516</span>              + initialServer.getHostname() + " to " + newServer.getHostname());<a name="line.516"></a>
+<span class="sourceLineNo">517</span>        }<a name="line.517"></a>
+<span class="sourceLineNo">518</span>        RegionPlan rp = new RegionPlan(region, initialServer, newServer);<a name="line.518"></a>
+<span class="sourceLineNo">519</span>        plans.add(rp);<a name="line.519"></a>
+<span class="sourceLineNo">520</span>      }<a name="line.520"></a>
+<span class="sourceLineNo">521</span>    }<a name="line.521"></a>
+<span class="sourceLineNo">522</span>    return plans;<a name="line.522"></a>
+<span class="sourceLineNo">523</span>  }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span>  /**<a name="line.525"></a>
+<span class="sourceLineNo">526</span>   * Store the current region loads.<a name="line.526"></a>
+<span class="sourceLineNo">527</span>   */<a name="line.527"></a>
+<span class="sourceLineNo">528</span>  private synchronized void updateRegionLoad() {<a name="line.528"></a>
+<span class="sourceLineNo">529</span>    // We create a new hashmap so that regions that are no longer there are removed.<a name="line.529"></a>
+<span class="sourceLineNo">530</span>    // However we temporarily need the old loads so we can use them to keep the rolling average.<a name="line.530"></a>
+<span class="sourceLineNo">531</span>    Map&lt;String, Deque&lt;BalancerRegionLoad&gt;&gt; oldLoads = loads;<a name="line.531"></a>
+<span class="sourceLineNo">532</span>    loads = new HashMap&lt;&gt;();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span>    clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -&gt; {<a name="line.534"></a>
+<span class="sourceLineNo">535</span>      sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -&gt; {<a name="line.535"></a>
+<span class="sourceLineNo">536</span>        Deque&lt;BalancerRegionLoad&gt; rLoads = oldLoads.get(Bytes.toString(regionName));<a name="line.536"></a>
+<span class="sourceLineNo">537</span>        if (rLoads == null) {<a name="line.537"></a>
+<span class="sourceLineNo">538</span>          rLoads = new ArrayDeque&lt;&gt;(numRegionLoadsToRemember + 1);<a name="line.538"></a>
+<span class="sourceLineNo">539</span>        } else if (rLoads.size() &gt;= numRegionLoadsToRemember) {<a name="line.539"></a>
+<span class="sourceLineNo">540</span>          rLoads.remove();<a name="line.540"></a>
+<span class="sourceLineNo">541</span>        }<a name="line.541"></a>
+<span class="sourceLineNo">542</span>        rLoads.add(new BalancerRegionLoad(rm));<a name="line.542"></a>
+<span class="sourceLineNo">543</span>        loads.put(Bytes.toString(regionName), rLoads);<a name="line.543"></a>
+<span class="sourceLineNo">544</span>      });<a name="line.544"></a>
+<span class="sourceLineNo">545</span>    });<a name="line.545"></a>
+<span class="sourceLineNo">546</span><a name="line.546"></a>
+<span class="sourceLineNo">547</span>    for(CostFromRegionLoadFunction cost : regionLoadFunctions) {<a name="line.547"></a>
+<span class="sourceLineNo">548</span>      cost.setLoads(loads);<a name="line.548"></a>
+<span class="sourceLineNo">549</span>    }<a name="line.549"></a>
+<span class="sourceLineNo">550</span>  }<a name="line.550"></a>
+<span class="sourceLineNo">551</span><a name="line.551"></a>
+<span class="sourceLineNo">552</span>  protected void initCosts(Cluster cluster) {<a name="line.552"></a>
+<span class="sourceLineNo">553</span>    for (CostFunction c:costFunctions) {<a name="line.553"></a>
+<span class="sourceLineNo">554</span>      c.init(cluster);<a name="line.554"></a>
+<span class="sourceLineNo">555</span>    }<a name="line.555"></a>
+<span class="sourceLineNo">556</span>  }<a name="line.556"></a>
+<span class="sourceLineNo">557</span><a name="line.557"></a>
+<span class="sourceLineNo">558</span>  protected void updateCostsWithAction(Cluster cluster, Action action) {<a name="line.558"></a>
+<span class="sourceLineNo">559</span>    for (CostFunction c : costFunctions) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span>      c.postAction(action);<a name="line.560"></a>
+<span class="sourceLineNo">561</span>    }<a name="line.561"></a>
+<span class="sourceLineNo">562</span>  }<a name="line.562"></a>
+<span class="sourceLineNo">563</span><a name="line.563"></a>
+<span class="sourceLineNo">564</span>  /**<a name="line.564"></a>
+<span class="sourceLineNo">565</span>   * Get the names of the cost functions<a name="line.565"></a>
+<span class="sourceLineNo">566</span>   */<a name="line.566"></a>
+<span class="sourceLineNo">567</span>  public String[] getCostFunctionNames() {<a name="line.567"></a>
+<span class="sourceLineNo">568</span>    if (costFunctions == null) return null;<a name="line.568"></a>
+<span class="sourceLineNo">569</span>    String[] ret = new String[costFunctions.length];<a name="line.569"></a>
+<span class="sourceLineNo">570</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.570"></a>
+<span class="sourceLineNo">571</span>      CostFunction c = costFunctions[i];<a name="line.571"></a>
+<span class="sourceLineNo">572</span>      ret[i] = c.getClass().getSimpleName();<a name="line.572"></a>
+<span class="sourceLineNo">573</span>    }<a name="line.573"></a>
+<span class="sourceLineNo">574</span><a name="line.574"></a>
+<span class="sourceLineNo">575</span>    return ret;<a name="line.575"></a>
+<span class="sourceLineNo">576</span>  }<a name="line.576"></a>
+<span class="sourceLineNo">577</span><a name="line.577"></a>
+<span class="sourceLineNo">578</span>  /**<a name="line.578"></a>
+<span class="sourceLineNo">579</span>   * This is the main cost function.  It will compute a cost associated with a proposed cluster<a name="line.579"></a>
+<span class="sourceLineNo">580</span>   * state.  All different costs will be combined with their multipliers to produce a double cost.<a name="line.580"></a>
+<span class="sourceLineNo">581</span>   *<a name="line.581"></a>
+<span class="sourceLineNo">582</span>   * @param cluster The state of the cluster<a name="line.582"></a>
+<span class="sourceLineNo">583</span>   * @param previousCost the previous cost. This is used as an early out.<a name="line.583"></a>
+<span class="sourceLineNo">584</span>   * @return a double of a cost associated with the proposed cluster state.  This cost is an<a name="line.584"></a>
+<span class="sourceLineNo">585</span>   *         aggregate of all individual cost functions.<a name="line.585"></a>
+<span class="sourceLineNo">586</span>   */<a name="line.586"></a>
+<span class="sourceLineNo">587</span>  protected double computeCost(Cluster cluster, double previousCost) {<a name="line.587"></a>
+<span class="sourceLineNo">588</span>    double total = 0;<a name="line.588"></a>
 <span class="sourceLineNo">589</span><a name="line.589"></a>
-<span class="sourceLineNo">590</span>      Float multiplier = c.getMultiplier();<a name="line.590"></a>
-<span class="sourceLineNo">591</span>      Double cost = c.cost();<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.593"></a>
-<span class="sourceLineNo">594</span>      total += this.tempFunctionCosts[i];<a name="line.594"></a>
-<span class="sourceLineNo">595</span><a name="line.595"></a>
-<span class="sourceLineNo">596</span>      if (total &gt; previousCost) {<a name="line.596"></a>
-<span class="sourceLineNo">597</span>        break;<a name="line.597"></a>
-<span class="sourceLineNo">598</span>      }<a name="line.598"></a>
-<span class="sourceLineNo">599</span>    }<a name="line.599"></a>
+<span class="sourceLineNo">590</span>    for (int i = 0; i &lt; costFunctions.length; i++) {<a name="line.590"></a>
+<span class="sourceLineNo">591</span>      CostFunction c = costFunctions[i];<a name="line.591"></a>
+<span class="sourceLineNo">592</span>      this.tempFunctionCosts[i] = 0.0;<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span>      if (c.getMultiplier() &lt;= 0) {<a name="line.594"></a>
+<span class="sourceLineNo">595</span>        continue;<a name="line.595"></a>
+<span class="sourceLineNo">596</span>      }<a name="line.596"></a>
+<span class="sourceLineNo">597</span><a name="line.597"></a>
+<span class="sourceLineNo">598</span>      Float multiplier = c.getMultiplier();<a name="line.598"></a>
+<span class="sourceLineNo">599</span>      Double cost = c.cost();<a name="line.599"></a>
 <span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span>    return total;<a name="line.601"></a>
-<span class="sourceLineNo">602</span>  }<a name="line.602"></a>
+<span class="sourceLineNo">601</span>      this.tempFunctionCosts[i] = multiplier*cost;<a name="line.601"></a>
+<span class="sourceLineNo">602</span>      total += this.tempFunctionCosts[i];<a name="line.602"></a>
 <span class="sourceLineNo">603</span><a name="line.603"></a>
-<span class="sourceLineNo">604</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.604"></a>
-<span class="sourceLineNo">605</span>  abstract static class CandidateGenerator {<a name="line.605"></a>
-<span class="sourceLineNo">606</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.606"></a>
-<span class="sourceLineNo">607</span><a name="line.607"></a>
-<span class="sourceLineNo">608</span>    /**<a name="line.608"></a>
-<span class="sourceLineNo">609</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.609"></a>
-<span class="sourceLineNo">610</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.610"></a>
-<span class="sourceLineNo">611</span>     * rather than swap.<a name="line.611"></a>
-<span class="sourceLineNo">612</span>     *<a name="line.612"></a>
-<span class="sourceLineNo">613</span>     * @param cluster        The state of the cluster<a name="line.613"></a>
-<span class="sourceLineNo">614</span>     * @param server         index of the server<a name="line.614"></a>
-<span class="sourceLineNo">615</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.615"></a>
-<span class="sourceLineNo">616</span>     *                       than a swap.<a name="line.616"></a>
-<span class="sourceLineNo">617</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.617"></a>
-<span class="sourceLineNo">618</span>     *         suggested.<a name="line.618"></a>
-<span class="sourceLineNo">619</span>     */<a name="line.619"></a>
-<span class="sourceLineNo">620</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span>      // Check to see if this is just a move.<a name="line.621"></a>
-<span class="sourceLineNo">622</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.622"></a>
-<span class="sourceLineNo">623</span>        // signal a move only.<a name="line.623"></a>
-<span class="sourceLineNo">624</span>        return -1;<a name="line.624"></a>
-<span class="sourceLineNo">625</span>      }<a name="line.625"></a>
-<span class="sourceLineNo">626</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.626"></a>
-<span class="sourceLineNo">627</span>      return cluster.regionsPerServer[server][rand];<a name="line.627"></a>
-<span class="sourceLineNo">628</span><a name="line.628"></a>
-<span class="sourceLineNo">629</span>    }<a name="line.629"></a>
-<span class="sourceLineNo">630</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.630"></a>
-<span class="sourceLineNo">631</span>      if (cluster.numServers &lt; 1) {<a name="line.631"></a>
+<span class="sourceLineNo">604</span>      if (total &gt; previousCost) {<a name="line.604"></a>
+<span class="sourceLineNo">605</span>        break;<a name="line.605"></a>
+<span class="sourceLineNo">606</span>      }<a name="line.606"></a>
+<span class="sourceLineNo">607</span>    }<a name="line.607"></a>
+<span class="sourceLineNo">608</span><a name="line.608"></a>
+<span class="sourceLineNo">609</span>    return total;<a name="line.609"></a>
+<span class="sourceLineNo">610</span>  }<a name="line.610"></a>
+<span class="sourceLineNo">611</span><a name="line.611"></a>
+<span class="sourceLineNo">612</span>  /** Generates a candidate action to be applied to the cluster for cost function search */<a name="line.612"></a>
+<span class="sourceLineNo">613</span>  abstract static class CandidateGenerator {<a name="line.613"></a>
+<span class="sourceLineNo">614</span>    abstract Cluster.Action generate(Cluster cluster);<a name="line.614"></a>
+<span class="sourceLineNo">615</span><a name="line.615"></a>
+<span class="sourceLineNo">616</span>    /**<a name="line.616"></a>
+<span class="sourceLineNo">617</span>     * From a list of regions pick a random one. Null can be returned which<a name="line.617"></a>
+<span class="sourceLineNo">618</span>     * {@link StochasticLoadBalancer#balanceCluster(Map)} recognize as signal to try a region move<a name="line.618"></a>
+<span class="sourceLineNo">619</span>     * rather than swap.<a name="line.619"></a>
+<span class="sourceLineNo">620</span>     *<a name="line.620"></a>
+<span class="sourceLineNo">621</span>     * @param cluster        The state of the cluster<a name="line.621"></a>
+<span class="sourceLineNo">622</span>     * @param server         index of the server<a name="line.622"></a>
+<span class="sourceLineNo">623</span>     * @param chanceOfNoSwap Chance that this will decide to try a move rather<a name="line.623"></a>
+<span class="sourceLineNo">624</span>     *                       than a swap.<a name="line.624"></a>
+<span class="sourceLineNo">625</span>     * @return a random {@link RegionInfo} or null if an asymmetrical move is<a name="line.625"></a>
+<span class="sourceLineNo">626</span>     *         suggested.<a name="line.626"></a>
+<span class="sourceLineNo">627</span>     */<a name="line.627"></a>
+<span class="sourceLineNo">628</span>    protected int pickRandomRegion(Cluster cluster, int server, double chanceOfNoSwap) {<a name="line.628"></a>
+<span class="sourceLineNo">629</span>      // Check to see if this is just a move.<a name="line.629"></a>
+<span class="sourceLineNo">630</span>      if (cluster.regionsPerServer[server].length == 0 || RANDOM.nextFloat() &lt; chanceOfNoSwap) {<a name="line.630"></a>
+<span class="sourceLineNo">631</span>        // signal a move only.<a name="line.631"></a>
 <span class="sourceLineNo">632</span>        return -1;<a name="line.632"></a>
 <span class="sourceLineNo">633</span>      }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.635"></a>
-<span class="sourceLineNo">636</span>    }<a name="line.636"></a>
-<span class="sourceLineNo">637</span><a name="line.637"></a>
-<span class="sourceLineNo">638</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.638"></a>
-<span class="sourceLineNo">639</span>      if (cluster.numRacks &lt; 1) {<a name="line.639"></a>
+<span class="sourceLineNo">634</span>      int rand = RANDOM.nextInt(cluster.regionsPerServer[server].length);<a name="line.634"></a>
+<span class="sourceLineNo">635</span>      return cluster.regionsPerServer[server][rand];<a name="line.635"></a>
+<span class="sourceLineNo">636</span><a name="line.636"></a>
+<span class="sourceLineNo">637</span>    }<a name="line.637"></a>
+<span class="sourceLineNo">638</span>    protected int pickRandomServer(Cluster cluster) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span>      if (cluster.numServers &lt; 1) {<a name="line.639"></a>
 <span class="sourceLineNo">640</span>        return -1;<a name="line.640"></a>
 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 <span class="sourceLineNo">642</span><a name="line.642"></a>
-<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.643"></a>
+<span class="sourceLineNo">643</span>      return RANDOM.nextInt(cluster.numServers);<a name="line.643"></a>
 <span class="sourceLineNo">644</span>    }<a name="line.644"></a>
 <span class="sourceLineNo">645</span><a name="line.645"></a>
-<span class="sourceLineNo">646</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.646"></a>
-<span class="sourceLineNo">647</span>      if (cluster.numServers &lt; 2) {<a name="line.647"></a>
+<span class="sourceLineNo">646</span>    protected int pickRandomRack(Cluster cluster) {<a name="line.646"></a>
+<span class="sourceLineNo">647</span>      if (cluster.numRacks &lt; 1) {<a name="line.647"></a>
 <span class="sourceLineNo">648</span>        return -1;<a name="line.648"></a>
 <span class="sourceLineNo">649</span>      }<a name="line.649"></a>
-<span class="sourceLineNo">650</span>      while (true) {<a name="line.650"></a>
-<span class="sourceLineNo">651</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.651"></a>
-<span class="sourceLineNo">652</span>        if (otherServerIndex != serverIndex) {<a name="line.652"></a>
-<span class="sourceLineNo">653</span>          return otherServerIndex;<a name="line.653"></a>
-<span class="sourceLineNo">654</span>        }<a name="line.654"></a>
-<span class="sourceLineNo">655</span>      }<a name="line.655"></a>
-<span class="sourceLineNo">656</span>    }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.658"></a>
-<span class="sourceLineNo">659</span>      if (cluster.numRacks &lt; 2) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span>        return -1;<a name="line.660"></a>
-<span class="sourceLineNo">661</span>      }<a name="line.661"></a>
-<span class="sourceLineNo">662</span>      while (true) {<a name="line.662"></a>
-<span class="sourceLineNo">663</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.663"></a>
-<span class="sourceLineNo">664</span>        if (otherRackIndex != rackIndex) {<a name="line.664"></a>
-<span class="sourceLineNo">665</span>          return otherRackIndex;<a name="line.665"></a>
-<span class="sourceLineNo">666</span>        }<a name="line.666"></a>
-<span class="sourceLineNo">667</span>      }<a name="line.667"></a>
-<span class="sourceLineNo">668</span>    }<a name="line.668"></a>
-<span class="sourceLineNo">669</span><a name="line.669"></a>
-<span class="sourceLineNo">670</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.670"></a>
-<span class="sourceLineNo">671</span>                                                       int thisServer,<a name="line.671"></a>
-<span class="sourceLineNo">672</span>                                                       int otherServer) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.673"></a>
-<span class="sourceLineNo">674</span>        return Cluster.NullAction;<a name="line.674"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span>      return RANDOM.nextInt(cluster.numRacks);<a name="line.651"></a>
+<span class="sourceLineNo">652</span>    }<a name="line.652"></a>
+<span class="sourceLineNo">653</span><a name="line.653"></a>
+<span class="sourceLineNo">654</span>    protected int pickOtherRandomServer(Cluster cluster, int serverIndex) {<a name="line.654"></a>
+<span class="sourceLineNo">655</span>      if (cluster.numServers &lt; 2) {<a name="line.655"></a>
+<span class="sourceLineNo">656</span>        return -1;<a name="line.656"></a>
+<span class="sourceLineNo">657</span>      }<a name="line.657"></a>
+<span class="sourceLineNo">658</span>      while (true) {<a name="line.658"></a>
+<span class="sourceLineNo">659</span>        int otherServerIndex = pickRandomServer(cluster);<a name="line.659"></a>
+<span class="sourceLineNo">660</span>        if (otherServerIndex != serverIndex) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span>          return otherServerIndex;<a name="line.661"></a>
+<span class="sourceLineNo">662</span>        }<a name="line.662"></a>
+<span class="sourceLineNo">663</span>      }<a name="line.663"></a>
+<span class="sourceLineNo">664</span>    }<a name="line.664"></a>
+<span class="sourceLineNo">665</span><a name="line.665"></a>
+<span class="sourceLineNo">666</span>    protected int pickOtherRandomRack(Cluster cluster, int rackIndex) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span>      if (cluster.numRacks &lt; 2) {<a name="line.667"></a>
+<span class="sourceLineNo">668</span>        return -1;<a name="line.668"></a>
+<span class="sourceLineNo">669</span>      }<a name="line.669"></a>
+<span class="sourceLineNo">670</span>      while (true) {<a name="line.670"></a>
+<span class="sourceLineNo">671</span>        int otherRackIndex = pickRandomRack(cluster);<a name="line.671"></a>
+<span class="sourceLineNo">672</span>        if (otherRackIndex != rackIndex) {<a name="line.672"></a>
+<span class="sourceLineNo">673</span>          return otherRackIndex;<a name="line.673"></a>
+<span class="sourceLineNo">674</span>        }<a name="line.674"></a>
 <span class="sourceLineNo">675</span>      }<a name="line.675"></a>
-<span class="sourceLineNo">676</span><a name="line.676"></a>
-<span class="sourceLineNo">677</span>      // Decide who is most likely to need another region<a name="line.677"></a>
-<span class="sourceLineNo">678</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.678"></a>
-<span class="sourceLineNo">679</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span>      // Assign the chance based upon the above<a name="line.681"></a>
-<span class="sourceLineNo">682</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.682"></a>
-<span class="sourceLineNo">683</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.683"></a>
+<span class="sourceLineNo">676</span>    }<a name="line.676"></a>
+<span class="sourceLineNo">677</span><a name="line.677"></a>
+<span class="sourceLineNo">678</span>    protected Cluster.Action pickRandomRegions(Cluster cluster,<a name="line.678"></a>
+<span class="sourceLineNo">679</span>                                                       int thisServer,<a name="line.679"></a>
+<span class="sourceLineNo">680</span>                                                       int otherServer) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span>      if (thisServer &lt; 0 || otherServer &lt; 0) {<a name="line.681"></a>
+<span class="sourceLineNo">682</span>        return Cluster.NullAction;<a name="line.682"></a>
+<span class="sourceLineNo">683</span>      }<a name="line.683"></a>
 <span class="sourceLineNo">684</span><a name="line.684"></a>
-<span class="sourceLineNo">685</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.685"></a>
-<span class="sourceLineNo">686</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.688"></a>
-<span class="sourceLineNo">689</span>    }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.691"></a>
-<span class="sourceLineNo">692</span>        int toServer, int toRegion) {<a name="line.692"></a>
-<span class="sourceLineNo">693</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.693"></a>
-<span class="sourceLineNo">694</span>        return Cluster.NullAction;<a name="line.694"></a>
-<span class="sourceLineNo">695</span>      }<a name="line.695"></a>
-<span class="sourceLineNo">696</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.696"></a>
-<span class="sourceLineNo">697</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.697"></a>
-<span class="sourceLineNo">698</span>          toServer, toRegion);<a name="line.698"></a>
-<span class="sourceLineNo">699</span>      } else if (fromRegion &gt; 0) {<a name="line.699"></a>
-<span class="sourceLineNo">700</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.700"></a>
-<span class="sourceLineNo">701</span>      } else if (toRegion &gt; 0) {<a name="line.701"></a>
-<span class="sourceLineNo">702</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.702"></a>
-<span class="sourceLineNo">703</span>      } else {<a name="line.703"></a>
-<span class="sourceLineNo">704</span>        return Cluster.NullAction;<a name="line.704"></a>
-<span class="sourceLineNo">705</span>      }<a name="line.705"></a>
-<span class="sourceLineNo">706</span>    }<a name="line.706"></a>
-<span class="sourceLineNo">707</span><a name="line.707"></a>
-<span class="sourceLineNo">708</span>    /**<a name="line.708"></a>
-<span class="sourceLineNo">709</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.709"></a>
-<span class="sourceLineNo">710</span>     */<a name="line.710"></a>
-<span class="sourceLineNo">711</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.711"></a>
-<span class="sourceLineNo">712</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.712"></a>
-<span class="sourceLineNo">713</span>      for (int i = 0; i &lt; length; i++) {<a name="line.713"></a>
-<span class="sourceLineNo">714</span>        order.add(i);<a name="line.714"></a>
-<span class="sourceLineNo">715</span>      }<a name="line.715"></a>
-<span class="sourceLineNo">716</span>      Collections.shuffle(order);<a name="line.716"></a>
-<span class="sourceLineNo">717</span>      return order;<a name="line.717"></a>
-<span class="sourceLineNo">718</span>    }<a name="line.718"></a>
-<span class="sourceLineNo">719</span>  }<a name="line.719"></a>
-<span class="sourceLineNo">720</span><a name="line.720"></a>
-<span class="sourceLineNo">721</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span>    @Override<a name="line.723"></a>
-<span class="sourceLineNo">724</span>    Cluster.Action generate(Cluster cluster) {<a name="line.724"></a>
-<span class="sourceLineNo">725</span><a name="line.725"></a>
-<span class="sourceLineNo">726</span>      int thisServer = pickRandomServer(cluster);<a name="line.726"></a>
-<span class="sourceLineNo">727</span><a name="line.727"></a>
-<span class="sourceLineNo">728</span>      // Pick the other server<a name="line.728"></a>
-<span class="sourceLineNo">729</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.729"></a>
+<span class="sourceLineNo">685</span>      // Decide who is most likely to need another region<a name="line.685"></a>
+<span class="sourceLineNo">686</span>      int thisRegionCount = cluster.getNumRegions(thisServer);<a name="line.686"></a>
+<span class="sourceLineNo">687</span>      int otherRegionCount = cluster.getNumRegions(otherServer);<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span>      // Assign the chance based upon the above<a name="line.689"></a>
+<span class="sourceLineNo">690</span>      double thisChance = (thisRegionCount &gt; otherRegionCount) ? 0 : 0.5;<a name="line.690"></a>
+<span class="sourceLineNo">691</span>      double otherChance = (thisRegionCount &lt;= otherRegionCount) ? 0 : 0.5;<a name="line.691"></a>
+<span class="sourceLineNo">692</span><a name="line.692"></a>
+<span class="sourceLineNo">693</span>      int thisRegion = pickRandomRegion(cluster, thisServer, thisChance);<a name="line.693"></a>
+<span class="sourceLineNo">694</span>      int otherRegion = pickRandomRegion(cluster, otherServer, otherChance);<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span>      return getAction(thisServer, thisRegion, otherServer, otherRegion);<a name="line.696"></a>
+<span class="sourceLineNo">697</span>    }<a name="line.697"></a>
+<span class="sourceLineNo">698</span><a name="line.698"></a>
+<span class="sourceLineNo">699</span>    protected Cluster.Action getAction(int fromServer, int fromRegion,<a name="line.699"></a>
+<span class="sourceLineNo">700</span>        int toServer, int toRegion) {<a name="line.700"></a>
+<span class="sourceLineNo">701</span>      if (fromServer &lt; 0 || toServer &lt; 0) {<a name="line.701"></a>
+<span class="sourceLineNo">702</span>        return Cluster.NullAction;<a name="line.702"></a>
+<span class="sourceLineNo">703</span>      }<a name="line.703"></a>
+<span class="sourceLineNo">704</span>      if (fromRegion &gt; 0 &amp;&amp; toRegion &gt; 0) {<a name="line.704"></a>
+<span class="sourceLineNo">705</span>        return new Cluster.SwapRegionsAction(fromServer, fromRegion,<a name="line.705"></a>
+<span class="sourceLineNo">706</span>          toServer, toRegion);<a name="line.706"></a>
+<span class="sourceLineNo">707</span>      } else if (fromRegion &gt; 0) {<a name="line.707"></a>
+<span class="sourceLineNo">708</span>        return new Cluster.MoveRegionAction(fromRegion, fromServer, toServer);<a name="line.708"></a>
+<span class="sourceLineNo">709</span>      } else if (toRegion &gt; 0) {<a name="line.709"></a>
+<span class="sourceLineNo">710</span>        return new Cluster.MoveRegionAction(toRegion, toServer, fromServer);<a name="line.710"></a>
+<span class="sourceLineNo">711</span>      } else {<a name="line.711"></a>
+<span class="sourceLineNo">712</span>        return Cluster.NullAction;<a name="line.712"></a>
+<span class="sourceLineNo">713</span>      }<a name="line.713"></a>
+<span class="sourceLineNo">714</span>    }<a name="line.714"></a>
+<span class="sourceLineNo">715</span><a name="line.715"></a>
+<span class="sourceLineNo">716</span>    /**<a name="line.716"></a>
+<span class="sourceLineNo">717</span>     * Returns a random iteration order of indexes of an array with size length<a name="line.717"></a>
+<span class="sourceLineNo">718</span>     */<a name="line.718"></a>
+<span class="sourceLineNo">719</span>    protected List&lt;Integer&gt; getRandomIterationOrder(int length) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span>      ArrayList&lt;Integer&gt; order = new ArrayList&lt;&gt;(length);<a name="line.720"></a>
+<span class="sourceLineNo">721</span>      for (int i = 0; i &lt; length; i++) {<a name="line.721"></a>
+<span class="sourceLineNo">722</span>        order.add(i);<a name="line.722"></a>
+<span class="sourceLineNo">723</span>      }<a name="line.723"></a>
+<span class="sourceLineNo">724</span>      Collections.shuffle(order);<a name="line.724"></a>
+<span class="sourceLineNo">725</span>      return order;<a name="line.725"></a>
+<span class="sourceLineNo">726</span>    }<a name="line.726"></a>
+<span class="sourceLineNo">727</span>  }<a name="line.727"></a>
+<span class="sourceLineNo">728</span><a name="line.728"></a>
+<span class="sourceLineNo">729</span>  static class RandomCandidateGenerator extends CandidateGenerator {<a name="line.729"></a>
 <span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.731"></a>
-<span class="sourceLineNo">732</span>    }<a name="line.732"></a>
-<span class="sourceLineNo">733</span>  }<a name="line.733"></a>
-<span class="sourceLineNo">734</span><a name="line.734"></a>
-<span class="sourceLineNo">735</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.735"></a>
-<span class="sourceLineNo">736</span><a name="line.736"></a>
-<span class="sourceLineNo">737</span>    @Override<a name="line.737"></a>
-<span class="sourceLineNo">738</span>    Cluster.Action generate(Cluster cluster) {<a name="line.738"></a>
-<span class="sourceLineNo">739</span>      cluster.sortServersByRegionCount();<a name="line.739"></a>
-<span class="sourceLineNo">740</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.740"></a>
-<span class="sourceLineNo">741</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.741"></a>
+<span class="sourceLineNo">731</span>    @Override<a name="line.731"></a>
+<span class="sourceLineNo">732</span>    Cluster.Action generate(Cluster cluster) {<a name="line.732"></a>
+<span class="sourceLineNo">733</span><a name="line.733"></a>
+<span class="sourceLineNo">734</span>      int thisServer = pickRandomServer(cluster);<a name="line.734"></a>
+<span class="sourceLineNo">735</span><a name="line.735"></a>
+<span class="sourceLineNo">736</span>      // Pick the other server<a name="line.736"></a>
+<span class="sourceLineNo">737</span>      int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.737"></a>
+<span class="sourceLineNo">738</span><a name="line.738"></a>
+<span class="sourceLineNo">739</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.739"></a>
+<span class="sourceLineNo">740</span>    }<a name="line.740"></a>
+<span class="sourceLineNo">741</span>  }<a name="line.741"></a>
 <span class="sourceLineNo">742</span><a name="line.742"></a>
-<span class="sourceLineNo">743</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.743"></a>
-<span class="sourceLineNo">744</span>    }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.746"></a>
-<span class="sourceLineNo">747</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.747"></a>
-<span class="sourceLineNo">748</span><a name="line.748"></a>
-<span class="sourceLineNo">749</span>      int index = 0;<a name="line.749"></a>
-<span class="sourceLineNo">750</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span>        index++;<a name="line.751"></a>
-<span class="sourceLineNo">752</span>        if (index == servers.length) {<a name="line.752"></a>
-<span class="sourceLineNo">753</span>          return -1;<a name="line.753"></a>
-<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
-<span class="sourceLineNo">755</span>      }<a name="line.755"></a>
-<span class="sourceLineNo">756</span>      return servers[index];<a name="line.756"></a>
-<span class="sourceLineNo">757</span>    }<a name="line.757"></a>
-<span class="sourceLineNo">758</span><a name="line.758"></a>
-<span class="sourceLineNo">759</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.760"></a>
-<span class="sourceLineNo">761</span><a name="line.761"></a>
-<span class="sourceLineNo">762</span>      int index = servers.length - 1;<a name="line.762"></a>
-<span class="sourceLineNo">763</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span>        index--;<a name="line.764"></a>
-<span class="sourceLineNo">765</span>        if (index &lt; 0) {<a name="line.765"></a>
-<span class="sourceLineNo">766</span>          return -1;<a name="line.766"></a>
-<span class="sourceLineNo">767</span>        }<a name="line.767"></a>
-<span class="sourceLineNo">768</span>      }<a name="line.768"></a>
-<span class="sourceLineNo">769</span>      return servers[index];<a name="line.769"></a>
-<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
-<span class="sourceLineNo">771</span>  }<a name="line.771"></a>
-<span class="sourceLineNo">772</span><a name="line.772"></a>
-<span class="sourceLineNo">773</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.773"></a>
-<span class="sourceLineNo">774</span><a name="line.774"></a>
-<span class="sourceLineNo">775</span>    private MasterServices masterServices;<a name="line.775"></a>
-<span class="sourceLineNo">776</span><a name="line.776"></a>
-<span class="sourceLineNo">777</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.777"></a>
-<span class="sourceLineNo">778</span>      this.masterServices = masterServices;<a name="line.778"></a>
-<span class="sourceLineNo">779</span>    }<a name="line.779"></a>
+<span class="sourceLineNo">743</span>  static class LoadCandidateGenerator extends CandidateGenerator {<a name="line.743"></a>
+<span class="sourceLineNo">744</span><a name="line.744"></a>
+<span class="sourceLineNo">745</span>    @Override<a name="line.745"></a>
+<span class="sourceLineNo">746</span>    Cluster.Action generate(Cluster cluster) {<a name="line.746"></a>
+<span class="sourceLineNo">747</span>      cluster.sortServersByRegionCount();<a name="line.747"></a>
+<span class="sourceLineNo">748</span>      int thisServer = pickMostLoadedServer(cluster, -1);<a name="line.748"></a>
+<span class="sourceLineNo">749</span>      int otherServer = pickLeastLoadedServer(cluster, thisServer);<a name="line.749"></a>
+<span class="sourceLineNo">750</span><a name="line.750"></a>
+<span class="sourceLineNo">751</span>      return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.751"></a>
+<span class="sourceLineNo">752</span>    }<a name="line.752"></a>
+<span class="sourceLineNo">753</span><a name="line.753"></a>
+<span class="sourceLineNo">754</span>    private int pickLeastLoadedServer(final Cluster cluster, int thisServer) {<a name="line.754"></a>
+<span class="sourceLineNo">755</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span>      int index = 0;<a name="line.757"></a>
+<span class="sourceLineNo">758</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.758"></a>
+<span class="sourceLineNo">759</span>        index++;<a name="line.759"></a>
+<span class="sourceLineNo">760</span>        if (index == servers.length) {<a name="line.760"></a>
+<span class="sourceLineNo">761</span>          return -1;<a name="line.761"></a>
+<span class="sourceLineNo">762</span>        }<a name="line.762"></a>
+<span class="sourceLineNo">763</span>      }<a name="line.763"></a>
+<span class="sourceLineNo">764</span>      return servers[index];<a name="line.764"></a>
+<span class="sourceLineNo">765</span>    }<a name="line.765"></a>
+<span class="sourceLineNo">766</span><a name="line.766"></a>
+<span class="sourceLineNo">767</span>    private int pickMostLoadedServer(final Cluster cluster, int thisServer) {<a name="line.767"></a>
+<span class="sourceLineNo">768</span>      Integer[] servers = cluster.serverIndicesSortedByRegionCount;<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span>      int index = servers.length - 1;<a name="line.770"></a>
+<span class="sourceLineNo">771</span>      while (servers[index] == null || servers[index] == thisServer) {<a name="line.771"></a>
+<span class="sourceLineNo">772</span>        index--;<a name="line.772"></a>
+<span class="sourceLineNo">773</span>        if (index &lt; 0) {<a name="line.773"></a>
+<span class="sourceLineNo">774</span>          return -1;<a name="line.774"></a>
+<span class="sourceLineNo">775</span>        }<a name="line.775"></a>
+<span class="sourceLineNo">776</span>      }<a name="line.776"></a>
+<span class="sourceLineNo">777</span>      return servers[index];<a name="line.777"></a>
+<span class="sourceLineNo">778</span>    }<a name="line.778"></a>
+<span class="sourceLineNo">779</span>  }<a name="line.779"></a>
 <span class="sourceLineNo">780</span><a name="line.780"></a>
-<span class="sourceLineNo">781</span>    @Override<a name="line.781"></a>
-<span class="sourceLineNo">782</span>    Cluster.Action generate(Cluster cluster) {<a name="line.782"></a>
-<span class="sourceLineNo">783</span>      if (this.masterServices == null) {<a name="line.783"></a>
-<span class="sourceLineNo">784</span>        int thisServer = pickRandomServer(cluster);<a name="line.784"></a>
-<span class="sourceLineNo">785</span>        // Pick the other server<a name="line.785"></a>
-<span class="sourceLineNo">786</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.786"></a>
-<span class="sourceLineNo">787</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.787"></a>
-<span class="sourceLineNo">788</span>      }<a name="line.788"></a>
-<span class="sourceLineNo">789</span><a name="line.789"></a>
-<span class="sourceLineNo">790</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.790"></a>
-<span class="sourceLineNo">791</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.791"></a>
-<span class="sourceLineNo">792</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.792"></a>
-<span class="sourceLineNo">793</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.793"></a>
-<span class="sourceLineNo">794</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.794"></a>
-<span class="sourceLineNo">795</span>              cluster,<a name="line.795"></a>
-<span class="sourceLineNo">796</span>              currentServer,<a name="line.796"></a>
-<span class="sourceLineNo">797</span>              region,<a name="line.797"></a>
-<span class="sourceLineNo">798</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.798"></a>
-<span class="sourceLineNo">799</span>          );<a name="line.799"></a>
-<span class="sourceLineNo">800</span>          if (potential.isPresent()) {<a name="line.800"></a>
-<span class="sourceLineNo">801</span>            return potential.get();<a name="line.801"></a>
-<span class="sourceLineNo">802</span>          }<a name="line.802"></a>
-<span class="sourceLineNo">803</span>        }<a name="line.803"></a>
-<span class="sourceLineNo">804</span>      }<a name="line.804"></a>
-<span class="sourceLineNo">805</span>      return Cluster.NullAction;<a name="line.805"></a>
-<span class="sourceLineNo">806</span>    }<a name="line.806"></a>
-<span class="sourceLineNo">807</span><a name="line.807"></a>
-<span class="sourceLineNo">808</span>    /**<a name="line.808"></a>
-<span class="sourceLineNo">809</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.809"></a>
-<span class="sourceLineNo">810</span>     * Returns empty optional if no move can be found<a name="line.810"></a>
-<span class="sourceLineNo">811</span>     */<a name="line.811"></a>
-<span class="sourceLineNo">812</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.812"></a>
-<span class="sourceLineNo">813</span>                                           int fromServer,<a name="line.813"></a>
-<span class="sourceLineNo">814</span>                                           int fromRegion,<a name="line.814"></a>
-<span class="sourceLineNo">815</span>                                           int toServer) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.816"></a>
-<span class="sourceLineNo">817</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.817"></a>
-<span class="sourceLineNo">818</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.818"></a>
-<span class="sourceLineNo">819</span>      }<a name="line.819"></a>
-<span class="sourceLineNo">820</span><a name="line.820"></a>
-<span class="sourceLineNo">821</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.821"></a>
-<span class="sourceLineNo">822</span>      double fromRegionLocalityDelta =<a name="line.822"></a>
-<span class="sourceLineNo">823</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.823"></a>
-<span class="sourceLineNo">824</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.825"></a>
-<span class="sourceLineNo">826</span>        double toRegionLocalityDelta =<a name="line.826"></a>
-<span class="sourceLineNo">827</span>            getWeightedLocality(cluster, toRegion, fromServer) - getWeightedLocality(cluster, toRegion, toServer);<a name="line.827"></a>
-<span class="sourceLineNo">828</span>        // If locality would remain neutral or improve, attempt the swap<a name="line.828"></a>
-<span class="sourceLineNo">829</span>        if (fromRegionLocalityDelta + toRegionLocalityDelta &gt;= 0) {<a name="line.829"></a>
-<span class="sourceLineNo">830</span>          return Optional.of(getAction(fromServer, fromRegion, toServer, toRegion));<a name="line.830"></a>
-<span class="sourceLineNo">831</span>        }<a name="line.831"></a>
-<span class="sourceLineNo">832</span>      }<a name="line.832"></a>
-<span class="sourceLineNo">833</span><a name="line.833"></a>
-<span class="sourceLineNo">834</span>      return Optional.absent();<a name="line.834"></a>
-<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
-<span class="sourceLineNo">836</span><a name="line.836"></a>
-<span class="sourceLineNo">837</span>    private double getWeightedLocality(Cluster cluster, int region, int server) {<a name="line.837"></a>
-<span class="sourceLineNo">838</span>      return cluster.getOrComputeWeightedLocality(region, server, LocalityType.SERVER);<a name="line.838"></a>
-<span class="sourceLineNo">839</span>    }<a name="line.839"></a>
-<span class="sourceLineNo">840</span><a name="line.840"></a>
-<span class="sourceLineNo">841</span>    void setServices(MasterServices services) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span>      this.masterServices = services;<a name="line.842"></a>
+<span class="sourceLineNo">781</span>  static class LocalityBasedCandidateGenerator extends CandidateGenerator {<a name="line.781"></a>
+<span class="sourceLineNo">782</span><a name="line.782"></a>
+<span class="sourceLineNo">783</span>    private MasterServices masterServices;<a name="line.783"></a>
+<span class="sourceLineNo">784</span><a name="line.784"></a>
+<span class="sourceLineNo">785</span>    LocalityBasedCandidateGenerator(MasterServices masterServices) {<a name="line.785"></a>
+<span class="sourceLineNo">786</span>      this.masterServices = masterServices;<a name="line.786"></a>
+<span class="sourceLineNo">787</span>    }<a name="line.787"></a>
+<span class="sourceLineNo">788</span><a name="line.788"></a>
+<span class="sourceLineNo">789</span>    @Override<a name="line.789"></a>
+<span class="sourceLineNo">790</span>    Cluster.Action generate(Cluster cluster) {<a name="line.790"></a>
+<span class="sourceLineNo">791</span>      if (this.masterServices == null) {<a name="line.791"></a>
+<span class="sourceLineNo">792</span>        int thisServer = pickRandomServer(cluster);<a name="line.792"></a>
+<span class="sourceLineNo">793</span>        // Pick the other server<a name="line.793"></a>
+<span class="sourceLineNo">794</span>        int otherServer = pickOtherRandomServer(cluster, thisServer);<a name="line.794"></a>
+<span class="sourceLineNo">795</span>        return pickRandomRegions(cluster, thisServer, otherServer);<a name="line.795"></a>
+<span class="sourceLineNo">796</span>      }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span>      // Randomly iterate through regions until you find one that is not on ideal host<a name="line.798"></a>
+<span class="sourceLineNo">799</span>      for (int region : getRandomIterationOrder(cluster.numRegions)) {<a name="line.799"></a>
+<span class="sourceLineNo">800</span>        int currentServer = cluster.regionIndexToServerIndex[region];<a name="line.800"></a>
+<span class="sourceLineNo">801</span>        if (currentServer != cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]) {<a name="line.801"></a>
+<span class="sourceLineNo">802</span>          Optional&lt;Action&gt; potential = tryMoveOrSwap(<a name="line.802"></a>
+<span class="sourceLineNo">803</span>              cluster,<a name="line.803"></a>
+<span class="sourceLineNo">804</span>              currentServer,<a name="line.804"></a>
+<span class="sourceLineNo">805</span>              region,<a name="line.805"></a>
+<span class="sourceLineNo">806</span>              cluster.getOrComputeRegionsToMostLocalEntities(LocalityType.SERVER)[region]<a name="line.806"></a>
+<span class="sourceLineNo">807</span>          );<a name="line.807"></a>
+<span class="sourceLineNo">808</span>          if (potential.isPresent()) {<a name="line.808"></a>
+<span class="sourceLineNo">809</span>            return potential.get();<a name="line.809"></a>
+<span class="sourceLineNo">810</span>          }<a name="line.810"></a>
+<span class="sourceLineNo">811</span>        }<a name="line.811"></a>
+<span class="sourceLineNo">812</span>      }<a name="line.812"></a>
+<span class="sourceLineNo">813</span>      return Cluster.NullAction;<a name="line.813"></a>
+<span class="sourceLineNo">814</span>    }<a name="line.814"></a>
+<span class="sourceLineNo">815</span><a name="line.815"></a>
+<span class="sourceLineNo">816</span>    /**<a name="line.816"></a>
+<span class="sourceLineNo">817</span>     * Try to generate a move/swap fromRegion between fromServer and toServer such that locality is improved.<a name="line.817"></a>
+<span class="sourceLineNo">818</span>     * Returns empty optional if no move can be found<a name="line.818"></a>
+<span class="sourceLineNo">819</span>     */<a name="line.819"></a>
+<span class="sourceLineNo">820</span>    private Optional&lt;Action&gt; tryMoveOrSwap(Cluster cluster,<a name="line.820"></a>
+<span class="sourceLineNo">821</span>                                           int fromServer,<a name="line.821"></a>
+<span class="sourceLineNo">822</span>                                           int fromRegion,<a name="line.822"></a>
+<span class="sourceLineNo">823</span>                                           int toServer) {<a name="line.823"></a>
+<span class="sourceLineNo">824</span>      // Try move first. We know apriori fromRegion has the highest locality on toServer<a name="line.824"></a>
+<span class="sourceLineNo">825</span>      if (cluster.serverHasTooFewRegions(toServer)) {<a name="line.825"></a>
+<span class="sourceLineNo">826</span>        return Optional.of(getAction(fromServer, fromRegion, toServer, -1));<a name="line.826"></a>
+<span class="sourceLineNo">827</span>      }<a name="line.827"></a>
+<span class="sourceLineNo">828</span><a name="line.828"></a>
+<span class="sourceLineNo">829</span>      // Compare locality gain/loss from swapping fromRegion with regions on toServer<a name="line.829"></a>
+<span class="sourceLineNo">830</span>      double fromRegionLocalityDelta =<a name="line.830"></a>
+<span class="sourceLineNo">831</span>          getWeightedLocality(cluster, fromRegion, toServer) - getWeightedLocality(cluster, fromRegion, fromServer);<a name="line.831"></a>
+<span class="sourceLineNo">832</span>      for (int toRegionIndex : getRandomIterationOrder(cluster.regionsPerServer[toServer].length)) {<a name="line.832"></a>
+<span class="sourceLineNo">833</span>        int toRegion = cluster.regionsPerServer[toServer][toRegionIndex];<a name="line.833"></a>
+<span class="sourceLineNo">834</span>        double toRegionLocalityDelta =<a name="line.834"></a>
+<span class="sourceLineNo">835</span>     

<TRUNCATED>

[51/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.


Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/8f09a71d
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/8f09a71d
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/8f09a71d

Branch: refs/heads/asf-site
Commit: 8f09a71db4350eb46b3ee302c9750b2f8293f4df
Parents: 02f92ca
Author: jenkins <bu...@apache.org>
Authored: Wed Oct 24 14:54:17 2018 +0000
Committer: jenkins <bu...@apache.org>
Committed: Wed Oct 24 14:54:17 2018 +0000

----------------------------------------------------------------------
 acid-semantics.html                             |    4 +-
 apache_hbase_reference_guide.pdf                |    4 +-
 book.html                                       |    2 +-
 bulk-loads.html                                 |    4 +-
 checkstyle-aggregate.html                       |  372 +-
 checkstyle.rss                                  |   16 +-
 coc.html                                        |    4 +-
 dependencies.html                               |    4 +-
 dependency-convergence.html                     |    4 +-
 dependency-info.html                            |    4 +-
 dependency-management.html                      |    4 +-
 devapidocs/constant-values.html                 |    4 +-
 devapidocs/index-all.html                       |   28 +-
 .../hadoop/hbase/backup/package-tree.html       |    2 +-
 .../hadoop/hbase/client/package-tree.html       |   22 +-
 .../hadoop/hbase/coprocessor/package-tree.html  |    2 +-
 .../hadoop/hbase/executor/package-tree.html     |    2 +-
 .../hadoop/hbase/filter/package-tree.html       |    8 +-
 .../hadoop/hbase/io/hfile/package-tree.html     |    6 +-
 .../apache/hadoop/hbase/ipc/package-tree.html   |    2 +-
 .../hadoop/hbase/mapreduce/package-tree.html    |    4 +-
 ...talogJanitor.SplitParentFirstComparator.html |    8 +-
 .../hadoop/hbase/master/CatalogJanitor.html     |   18 +-
 .../master/HMaster.MasterStoppedException.html  |    4 +-
 .../master/HMaster.TableDescriptorGetter.html   |    4 +-
 .../org/apache/hadoop/hbase/master/HMaster.html |  256 +-
 ...asticLoadBalancer.CPRequestCostFunction.html |   10 +-
 ...ochasticLoadBalancer.CandidateGenerator.html |   22 +-
 ...lancer.CostFromRegionLoadAsRateFunction.html |    6 +-
 ...LoadBalancer.CostFromRegionLoadFunction.html |   20 +-
 .../StochasticLoadBalancer.CostFunction.html    |   28 +-
 ...sticLoadBalancer.LoadCandidateGenerator.html |   10 +-
 ...alancer.LocalityBasedCandidateGenerator.html |   14 +-
 ...cLoadBalancer.LocalityBasedCostFunction.html |   26 +-
 ...icLoadBalancer.MemStoreSizeCostFunction.html |   10 +-
 ...StochasticLoadBalancer.MoveCostFunction.html |   18 +-
 ...ncer.PrimaryRegionCountSkewCostFunction.html |   12 +-
 ...icLoadBalancer.RackLocalityCostFunction.html |   10 +-
 ...icLoadBalancer.RandomCandidateGenerator.html |    6 +-
 ...ticLoadBalancer.ReadRequestCostFunction.html |   10 +-
 ...oadBalancer.RegionCountSkewCostFunction.html |   12 +-
 ...alancer.RegionReplicaCandidateGenerator.html |   10 +-
 ...dBalancer.RegionReplicaHostCostFunction.html |   26 +-
 ...cer.RegionReplicaRackCandidateGenerator.html |    6 +-
 ...dBalancer.RegionReplicaRackCostFunction.html |   12 +-
 ...LoadBalancer.ServerLocalityCostFunction.html |   10 +-
 ...asticLoadBalancer.StoreFileCostFunction.html |   10 +-
 ...asticLoadBalancer.TableSkewCostFunction.html |   10 +-
 ...icLoadBalancer.WriteRequestCostFunction.html |   10 +-
 .../master/balancer/StochasticLoadBalancer.html |   18 +-
 .../hbase/master/balancer/package-tree.html     |    2 +-
 .../hadoop/hbase/master/package-tree.html       |    4 +-
 .../hbase/master/procedure/package-tree.html    |    2 +-
 .../org/apache/hadoop/hbase/package-tree.html   |   16 +-
 .../procedure2/RootProcedureState.State.html    |    8 +-
 .../hbase/procedure2/RootProcedureState.html    |   59 +-
 .../hadoop/hbase/procedure2/package-tree.html   |    6 +-
 .../hbase/procedure2/store/BitSetNode.html      |   48 +-
 .../ProcedureStoreTracker.DeleteState.html      |   12 +-
 .../procedure2/store/ProcedureStoreTracker.html |  107 +-
 .../procedure2/store/class-use/BitSetNode.html  |   14 +
 .../store/class-use/ProcedureStoreTracker.html  |   20 +-
 .../store/wal/ProcedureWALFormat.html           |   38 +-
 .../store/wal/ProcedureWALFormatReader.html     |   22 +-
 .../procedure2/store/wal/WALProcedureStore.html |   32 +-
 .../store/wal/WALProcedureTree.Iter.html        |   24 +-
 .../procedure2/store/wal/WALProcedureTree.html  |   14 +-
 .../store/wal/class-use/ProcedureWALFile.html   |    5 +-
 .../class-use/ProcedureWALFormat.Loader.html    |    5 +-
 .../hadoop/hbase/quotas/package-tree.html       |   10 +-
 ...eBulkLoadManager.SecureBulkLoadListener.html |   24 +-
 .../regionserver/SecureBulkLoadManager.html     |  144 +-
 .../hbase/regionserver/class-use/HRegion.html   |    8 +
 .../hadoop/hbase/regionserver/package-tree.html |   16 +-
 .../regionserver/querymatcher/package-tree.html |    4 +-
 .../hadoop/hbase/replication/package-tree.html  |    2 +-
 .../hbase/security/access/package-tree.html     |    2 +-
 .../hadoop/hbase/security/package-tree.html     |    4 +-
 .../hadoop/hbase/thrift/package-tree.html       |    2 +-
 .../apache/hadoop/hbase/util/package-tree.html  |   10 +-
 .../org/apache/hadoop/hbase/Version.html        |    4 +-
 ...talogJanitor.SplitParentFirstComparator.html |  696 +--
 .../hadoop/hbase/master/CatalogJanitor.html     |  696 +--
 .../master/HMaster.InitializationMonitor.html   | 4300 +++++++++---------
 .../master/HMaster.MasterStoppedException.html  | 4300 +++++++++---------
 .../hbase/master/HMaster.RedirectServlet.html   | 4300 +++++++++---------
 .../master/HMaster.TableDescriptorGetter.html   | 4300 +++++++++---------
 .../org/apache/hadoop/hbase/master/HMaster.html | 4300 +++++++++---------
 ...asticLoadBalancer.CPRequestCostFunction.html | 2630 +++++------
 ...ochasticLoadBalancer.CandidateGenerator.html | 2630 +++++------
 ...lancer.CostFromRegionLoadAsRateFunction.html | 2630 +++++------
 ...LoadBalancer.CostFromRegionLoadFunction.html | 2630 +++++------
 .../StochasticLoadBalancer.CostFunction.html    | 2630 +++++------
 ...sticLoadBalancer.LoadCandidateGenerator.html | 2630 +++++------
 ...alancer.LocalityBasedCandidateGenerator.html | 2630 +++++------
 ...cLoadBalancer.LocalityBasedCostFunction.html | 2630 +++++------
 ...icLoadBalancer.MemStoreSizeCostFunction.html | 2630 +++++------
 ...StochasticLoadBalancer.MoveCostFunction.html | 2630 +++++------
 ...ncer.PrimaryRegionCountSkewCostFunction.html | 2630 +++++------
 ...icLoadBalancer.RackLocalityCostFunction.html | 2630 +++++------
 ...icLoadBalancer.RandomCandidateGenerator.html | 2630 +++++------
 ...ticLoadBalancer.ReadRequestCostFunction.html | 2630 +++++------
 ...oadBalancer.RegionCountSkewCostFunction.html | 2630 +++++------
 ...alancer.RegionReplicaCandidateGenerator.html | 2630 +++++------
 ...dBalancer.RegionReplicaHostCostFunction.html | 2630 +++++------
 ...cer.RegionReplicaRackCandidateGenerator.html | 2630 +++++------
 ...dBalancer.RegionReplicaRackCostFunction.html | 2630 +++++------
 ...LoadBalancer.ServerLocalityCostFunction.html | 2630 +++++------
 ...asticLoadBalancer.StoreFileCostFunction.html | 2630 +++++------
 ...asticLoadBalancer.TableSkewCostFunction.html | 2630 +++++------
 ...icLoadBalancer.WriteRequestCostFunction.html | 2630 +++++------
 .../master/balancer/StochasticLoadBalancer.html | 2630 +++++------
 .../procedure2/RootProcedureState.State.html    |  347 +-
 .../hbase/procedure2/RootProcedureState.html    |  347 +-
 .../hbase/procedure2/store/BitSetNode.html      |  570 +--
 .../ProcedureStoreTracker.DeleteState.html      |  416 +-
 .../procedure2/store/ProcedureStoreTracker.html |  416 +-
 ...cedureWALFormat.InvalidWALDataException.html |  369 +-
 .../store/wal/ProcedureWALFormat.Loader.html    |  369 +-
 .../store/wal/ProcedureWALFormat.html           |  369 +-
 .../store/wal/ProcedureWALFormatReader.html     |  187 +-
 .../wal/WALProcedureStore.LeaseRecovery.html    |  477 +-
 .../store/wal/WALProcedureStore.PushType.html   |  477 +-
 .../wal/WALProcedureStore.SyncMetrics.html      |  477 +-
 .../procedure2/store/wal/WALProcedureStore.html |  477 +-
 .../store/wal/WALProcedureTree.Entry.html       |  307 +-
 .../store/wal/WALProcedureTree.Iter.html        |  307 +-
 .../wal/WALProcedureTree.ProtoAndProc.html      |  307 +-
 .../procedure2/store/wal/WALProcedureTree.html  |  307 +-
 ...eBulkLoadManager.SecureBulkLoadListener.html |  799 ++--
 .../regionserver/SecureBulkLoadManager.html     |  799 ++--
 downloads.html                                  |    4 +-
 export_control.html                             |    4 +-
 index.html                                      |    4 +-
 integration.html                                |    4 +-
 issue-tracking.html                             |    4 +-
 license.html                                    |    4 +-
 mail-lists.html                                 |    4 +-
 metrics.html                                    |    4 +-
 old_news.html                                   |    4 +-
 plugin-management.html                          |    4 +-
 plugins.html                                    |    4 +-
 poweredbyhbase.html                             |    4 +-
 project-info.html                               |    4 +-
 project-reports.html                            |    4 +-
 project-summary.html                            |    4 +-
 pseudo-distributed.html                         |    4 +-
 replication.html                                |    4 +-
 resources.html                                  |    4 +-
 source-repository.html                          |    4 +-
 sponsors.html                                   |    4 +-
 supportingprojects.html                         |    4 +-
 team-list.html                                  |    4 +-
 testdevapidocs/allclasses-frame.html            |    3 +-
 testdevapidocs/allclasses-noframe.html          |    3 +-
 testdevapidocs/constant-values.html             |    2 +-
 testdevapidocs/index-all.html                   |   81 +-
 .../hbase/class-use/HBaseClassTestRule.html     |   46 +-
 .../hbase/class-use/HBaseTestingUtility.html    |   40 +-
 .../hadoop/hbase/io/hfile/package-tree.html     |    2 +-
 .../TestMergeTableRegionsProcedure.html         |    8 +-
 .../org/apache/hadoop/hbase/package-tree.html   |   12 +-
 .../hadoop/hbase/procedure/package-tree.html    |    8 +-
 .../ProcedureTestingUtility.NoopProcedure.html  |    2 +-
 .../TestProcedureCleanup.ExchangeProcedure.html |  373 ++
 .../TestProcedureCleanup.RootProcedure.html     |   28 +-
 .../TestProcedureCleanup.SuspendProcedure.html  |   28 +-
 .../TestProcedureCleanup.TestProcEnv.html       |  244 -
 .../TestProcedureCleanup.WaitProcedure.html     |   28 +-
 .../hbase/procedure2/TestProcedureCleanup.html  |  124 +-
 .../ProcedureTestingUtility.NoopProcedure.html  |   18 +-
 .../TestProcedureCleanup.ExchangeProcedure.html |  125 +
 .../TestProcedureCleanup.TestProcEnv.html       |  199 -
 .../hadoop/hbase/procedure2/package-frame.html  |    2 +-
 .../hbase/procedure2/package-summary.html       |    6 +-
 .../hadoop/hbase/procedure2/package-tree.html   |    4 +-
 .../hadoop/hbase/procedure2/package-use.html    |   45 +-
 .../regionserver/TestSCVFWithMiniCluster.html   |    4 +-
 ...BulkLoadManager.MyExceptionToAvoidRetry.html |  286 ++
 .../regionserver/TestSecureBulkLoadManager.html |  604 +++
 .../regionserver/TestSeekOptimizations.html     |    4 +-
 ...BulkLoadManager.MyExceptionToAvoidRetry.html |  125 +
 .../class-use/TestSecureBulkLoadManager.html    |  125 +
 .../hbase/regionserver/package-frame.html       |    1 +
 .../hbase/regionserver/package-summary.html     |  109 +-
 .../hadoop/hbase/regionserver/package-tree.html |   26 +-
 .../apache/hadoop/hbase/wal/package-tree.html   |    2 +-
 testdevapidocs/overview-tree.html               |   12 +-
 .../TestMergeTableRegionsProcedure.html         |  125 +-
 .../TestProcedureCleanup.ExchangeProcedure.html |  358 ++
 .../TestProcedureCleanup.RootProcedure.html     |  480 +-
 .../TestProcedureCleanup.SuspendProcedure.html  |  480 +-
 .../TestProcedureCleanup.TestProcEnv.html       |  314 --
 .../TestProcedureCleanup.WaitProcedure.html     |  480 +-
 .../hbase/procedure2/TestProcedureCleanup.html  |  480 +-
 ...BulkLoadManager.MyExceptionToAvoidRetry.html |  320 ++
 .../regionserver/TestSecureBulkLoadManager.html |  320 ++
 197 files changed, 52436 insertions(+), 49660 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/acid-semantics.html
----------------------------------------------------------------------
diff --git a/acid-semantics.html b/acid-semantics.html
index bd92eb5..44293a2 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
       Apache HBase (TM) ACID Properties
@@ -611,7 +611,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/apache_hbase_reference_guide.pdf
----------------------------------------------------------------------
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index 9cd2adc..097d345 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
 /Author (Apache HBase Team)
 /Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2)
 /Producer (Apache HBase Team)
-/ModDate (D:20181023143419+00'00')
-/CreationDate (D:20181023145136+00'00')
+/ModDate (D:20181024143358+00'00')
+/CreationDate (D:20181024145050+00'00')
 >>
 endobj
 2 0 obj

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index aae5781..67ffe2c 100644
--- a/book.html
+++ b/book.html
@@ -41324,7 +41324,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
 <div id="footer">
 <div id="footer-text">
 Version 3.0.0-SNAPSHOT<br>
-Last updated 2018-10-23 14:34:19 UTC
+Last updated 2018-10-24 14:33:58 UTC
 </div>
 </div>
 </body>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index ec34ed4..1b2c6f1 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013;  
       Bulk Loads in Apache HBase (TM)
@@ -316,7 +316,7 @@ under the License. -->
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/checkstyle-aggregate.html
----------------------------------------------------------------------
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 34fbb96..07ba432 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Checkstyle Results</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -291,7 +291,7 @@
 <th><img src="images/icon_warning_sml.gif" alt="" />&#160;Warnings</th>
 <th><img src="images/icon_error_sml.gif" alt="" />&#160;Errors</th></tr>
 <tr class="b">
-<td>3797</td>
+<td>3798</td>
 <td>0</td>
 <td>0</td>
 <td>15098</td></tr></table></div>
@@ -59872,151 +59872,151 @@
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>133</td></tr>
+<td>135</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>148</td></tr>
+<td>150</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>164</td></tr>
+<td>166</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>167</td></tr>
+<td>169</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>173</td></tr>
+<td>175</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>174</td></tr>
+<td>176</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>193</td></tr>
+<td>195</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>196</td></tr>
+<td>198</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>232</td></tr>
+<td>234</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>311</td></tr>
+<td>313</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>312</td></tr>
+<td>314</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>315</td></tr>
+<td>317</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>318</td></tr>
+<td>320</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>332</td></tr>
+<td>334</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>333</td></tr>
+<td>335</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>336</td></tr>
+<td>338</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>368</td></tr>
+<td>370</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>369</td></tr>
+<td>371</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>382</td></tr>
+<td>384</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>383</td></tr>
+<td>385</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>384</td></tr>
+<td>386</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>387</td></tr>
+<td>389</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>coding</td>
 <td>InnerAssignment</td>
 <td>Inner assignments should be avoided.</td>
-<td>417</td></tr>
+<td>419</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>437</td></tr>
+<td>439</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>439</td></tr></table></div>
+<td>441</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.ClusterSchema.java">org/apache/hadoop/hbase/master/ClusterSchema.java</h3>
 <table border="0" class="table table-striped">
@@ -60370,745 +60370,745 @@
 <td>whitespace</td>
 <td>MethodParamPad</td>
 <td>'(' is preceded with whitespace.</td>
-<td>1864</td></tr>
+<td>1866</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>whitespace</td>
 <td>MethodParamPad</td>
 <td>'(' is preceded with whitespace.</td>
-<td>1868</td></tr>
+<td>1870</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1880</td></tr>
+<td>1882</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1882</td></tr>
+<td>1884</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1884</td></tr>
+<td>1886</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1887</td></tr>
+<td>1889</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1890</td></tr>
+<td>1892</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1891</td></tr>
+<td>1893</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1893</td></tr>
+<td>1895</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1895</td></tr>
+<td>1897</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1896</td></tr>
+<td>1898</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>1897</td></tr>
+<td>1899</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>1903</td></tr>
+<td>1905</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1907</td></tr>
+<td>1909</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1909</td></tr>
+<td>1911</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1910</td></tr>
+<td>1912</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1913</td></tr>
+<td>1915</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1914</td></tr>
+<td>1916</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1916</td></tr>
+<td>1918</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>1918</td></tr>
+<td>1920</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>1919</td></tr>
+<td>1921</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>1920</td></tr>
+<td>1922</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 116).</td>
-<td>1939</td></tr>
+<td>1941</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 105).</td>
-<td>1941</td></tr>
+<td>1943</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2032</td></tr>
+<td>2034</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2034</td></tr>
+<td>2036</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2036</td></tr>
+<td>2038</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2043</td></tr>
+<td>2045</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2044</td></tr>
+<td>2046</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2046</td></tr>
+<td>2048</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2048</td></tr>
+<td>2050</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2049</td></tr>
+<td>2051</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2051</td></tr>
+<td>2053</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2053</td></tr>
+<td>2055</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2054</td></tr>
+<td>2056</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2055</td></tr>
+<td>2057</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>2085</td></tr>
+<td>2087</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2347</td></tr>
+<td>2349</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2348</td></tr>
+<td>2350</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2355</td></tr>
+<td>2357</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2356</td></tr>
+<td>2358</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2362</td></tr>
+<td>2364</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2363</td></tr>
+<td>2365</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2370</td></tr>
+<td>2372</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>2371</td></tr>
+<td>2373</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2376</td></tr>
+<td>2378</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2394</td></tr>
+<td>2396</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2396</td></tr>
+<td>2398</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2398</td></tr>
+<td>2400</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2404</td></tr>
+<td>2406</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2405</td></tr>
+<td>2407</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2407</td></tr>
+<td>2409</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2409</td></tr>
+<td>2411</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2410</td></tr>
+<td>2412</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2412</td></tr>
+<td>2414</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2414</td></tr>
+<td>2416</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2415</td></tr>
+<td>2417</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2416</td></tr>
+<td>2418</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2429</td></tr>
+<td>2431</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2431</td></tr>
+<td>2433</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2433</td></tr>
+<td>2435</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2434</td></tr>
+<td>2436</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2435</td></tr>
+<td>2437</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2437</td></tr>
+<td>2439</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2439</td></tr>
+<td>2441</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2440</td></tr>
+<td>2442</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2442</td></tr>
+<td>2444</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2444</td></tr>
+<td>2446</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2445</td></tr>
+<td>2447</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2446</td></tr>
+<td>2448</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2529</td></tr>
+<td>2531</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2531</td></tr>
+<td>2533</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2537</td></tr>
+<td>2539</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2538</td></tr>
+<td>2540</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2539</td></tr>
+<td>2541</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2540</td></tr>
+<td>2542</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2541</td></tr>
+<td>2543</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2542</td></tr>
+<td>2544</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2543</td></tr>
+<td>2545</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 16, expected level should be one of the following: 18, 20.</td>
-<td>2544</td></tr>
+<td>2546</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 14, expected level should be one of the following: 16, 18.</td>
-<td>2546</td></tr>
+<td>2548</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2547</td></tr>
+<td>2549</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2548</td></tr>
+<td>2550</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if' child has incorrect indentation level 12, expected level should be one of the following: 14, 16.</td>
-<td>2549</td></tr>
+<td>2551</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 10, expected level should be one of the following: 12, 14.</td>
-<td>2550</td></tr>
+<td>2552</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'if rcurly' has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2551</td></tr>
+<td>2553</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2553</td></tr>
+<td>2555</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2560</td></tr>
+<td>2562</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2561</td></tr>
+<td>2563</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2563</td></tr>
+<td>2565</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2565</td></tr>
+<td>2567</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2566</td></tr>
+<td>2568</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2568</td></tr>
+<td>2570</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2570</td></tr>
+<td>2572</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2571</td></tr>
+<td>2573</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2572</td></tr>
+<td>2574</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2582</td></tr>
+<td>2584</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2584</td></tr>
+<td>2586</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2586</td></tr>
+<td>2588</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2596</td></tr>
+<td>2598</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2597</td></tr>
+<td>2599</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2599</td></tr>
+<td>2601</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2601</td></tr>
+<td>2603</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2602</td></tr>
+<td>2604</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2604</td></tr>
+<td>2606</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2606</td></tr>
+<td>2608</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2607</td></tr>
+<td>2609</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2608</td></tr>
+<td>2610</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>2620</td></tr>
+<td>2622</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2705</td></tr>
+<td>2707</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2709</td></tr>
+<td>2711</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def modifier' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2711</td></tr>
+<td>2713</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def' child has incorrect indentation level 8, expected level should be one of the following: 10, 12.</td>
-<td>2713</td></tr>
+<td>2715</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'method def rcurly' has incorrect indentation level 6, expected level should be one of the following: 8, 10.</td>
-<td>2714</td></tr>
+<td>2716</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'object def rcurly' has incorrect indentation level 4, expected level should be one of the following: 6, 8.</td>
-<td>2715</td></tr>
+<td>2717</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>3173</td></tr>
+<td>3175</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3349</td></tr>
+<td>3351</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3351</td></tr>
+<td>3353</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>3444</td></tr>
+<td>3446</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'for' construct must use '{}'s.</td>
-<td>3474</td></tr>
+<td>3476</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>3481</td></tr>
+<td>3483</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>3486</td></tr>
+<td>3488</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>3515</td></tr>
+<td>3517</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 111).</td>
-<td>3613</td></tr></table></div>
+<td>3615</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.HMasterCommandLine.java">org/apache/hadoop/hbase/master/HMasterCommandLine.java</h3>
 <table border="0" class="table table-striped">
@@ -65497,175 +65497,175 @@
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>453</td></tr>
+<td>461</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>560</td></tr>
+<td>568</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 107).</td>
-<td>793</td></tr>
+<td>801</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 109).</td>
-<td>809</td></tr>
+<td>817</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 116).</td>
-<td>823</td></tr>
+<td>831</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 114).</td>
-<td>827</td></tr>
+<td>835</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>JavadocTagContinuationIndentation</td>
 <td>Line continuation have incorrect indentation level, expected level should be 2.</td>
-<td>858</td></tr>
+<td>866</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'case' child has incorrect indentation level 6, expected level should be 8.</td>
-<td>999</td></tr>
+<td>1007</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'case' child has incorrect indentation level 6, expected level should be 8.</td>
-<td>1000</td></tr>
+<td>1008</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1001</td></tr>
+<td>1009</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1002</td></tr>
+<td>1010</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1003</td></tr>
+<td>1011</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'case' child has incorrect indentation level 6, expected level should be 8.</td>
-<td>1004</td></tr>
+<td>1012</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1005</td></tr>
+<td>1013</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1006</td></tr>
+<td>1014</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1007</td></tr>
+<td>1015</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'case' child has incorrect indentation level 6, expected level should be 8.</td>
-<td>1008</td></tr>
+<td>1016</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1009</td></tr>
+<td>1017</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1010</td></tr>
+<td>1018</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1011</td></tr>
+<td>1019</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1012</td></tr>
+<td>1020</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'case' child has incorrect indentation level 6, expected level should be 8.</td>
-<td>1013</td></tr>
+<td>1021</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>indentation</td>
 <td>Indentation</td>
 <td>'block' child has incorrect indentation level 8, expected level should be 10.</td>
-<td>1014</td></tr>
+<td>1022</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 119).</td>
-<td>1024</td></tr>
+<td>1032</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'if' construct must use '{}'s.</td>
-<td>1085</td></tr>
+<td>1093</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 106).</td>
-<td>1292</td></tr>
+<td>1300</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 106).</td>
-<td>1293</td></tr>
+<td>1301</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 109).</td>
-<td>1297</td></tr>
+<td>1305</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 109).</td>
-<td>1340</td></tr></table></div>
+<td>1348</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.master.balancer.TestBalancerStatusTagInJMXMetrics.java">org/apache/hadoop/hbase/master/balancer/TestBalancerStatusTagInJMXMetrics.java</h3>
 <table border="0" class="table table-striped">
@@ -71554,7 +71554,7 @@
 <td>blocks</td>
 <td>NeedBraces</td>
 <td>'while' construct must use '{}'s.</td>
-<td>179</td></tr></table></div>
+<td>184</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.procedure2.StateMachineProcedure.java">org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java</h3>
 <table border="0" class="table table-striped">
@@ -71896,7 +71896,7 @@
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 102).</td>
-<td>67</td></tr>
+<td>68</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>blocks</td>
@@ -81220,19 +81220,19 @@
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>397</td></tr>
+<td>440</td></tr>
 <tr class="a">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>javadoc</td>
 <td>NonEmptyAtclauseDescription</td>
 <td>At-clause should have a non-empty description.</td>
-<td>399</td></tr>
+<td>442</td></tr>
 <tr class="b">
 <td><img src="images/icon_error_sml.gif" alt="" />&#160;Error</td>
 <td>sizes</td>
 <td>LineLength</td>
 <td>Line is longer than 100 characters (found 101).</td>
-<td>405</td></tr></table></div>
+<td>448</td></tr></table></div>
 <div class="section">
 <h3 id="org.apache.hadoop.hbase.regionserver.Segment.java">org/apache/hadoop/hbase/regionserver/Segment.java</h3>
 <table border="0" class="table table-striped">
@@ -117407,7 +117407,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/checkstyle.rss
----------------------------------------------------------------------
diff --git a/checkstyle.rss b/checkstyle.rss
index e027e1c..236ac6d 100644
--- a/checkstyle.rss
+++ b/checkstyle.rss
@@ -25,7 +25,7 @@ under the License.
     <language>en-us</language>
     <copyright>&#169;2007 - 2018 The Apache Software Foundation</copyright>
     <item>
-      <title>File: 3797,
+      <title>File: 3798,
              Errors: 15098,
              Warnings: 0,
              Infos: 0
@@ -7816,6 +7816,20 @@ under the License.
               </tr>
                           <tr>
                 <td>
+                  <a href="http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.regionserver.TestSecureBulkLoadManager.java">org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java</a>
+                </td>
+                <td>
+                  0
+                </td>
+                <td>
+                  0
+                </td>
+                <td>
+                  0
+                </td>
+              </tr>
+                          <tr>
+                <td>
                   <a href="http://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.regionserver.TestMetricsRegionServer.java">org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java</a>
                 </td>
                 <td>

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/coc.html
----------------------------------------------------------------------
diff --git a/coc.html b/coc.html
index 7c3c028..5d8a01c 100644
--- a/coc.html
+++ b/coc.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; 
       Code of Conduct Policy
@@ -385,7 +385,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/dependencies.html
----------------------------------------------------------------------
diff --git a/dependencies.html b/dependencies.html
index 716b835..c96c14a 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Dependencies</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -450,7 +450,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/dependency-convergence.html
----------------------------------------------------------------------
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 309363f..a4020c7 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Reactor Dependency Convergence</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -889,7 +889,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/dependency-info.html
----------------------------------------------------------------------
diff --git a/dependency-info.html b/dependency-info.html
index aa0a365..48b526a 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Dependency Information</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -323,7 +323,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/dependency-management.html
----------------------------------------------------------------------
diff --git a/dependency-management.html b/dependency-management.html
index 5852a35..3895824 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -7,7 +7,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20181023" />
+    <meta name="Date-Revision-yyyymmdd" content="20181024" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Apache HBase &#x2013; Project Dependency Management</title>
     <link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -1015,7 +1015,7 @@
                         <a href="https://www.apache.org/">The Apache Software Foundation</a>.
             All rights reserved.      
                     
-                  <li id="publishDate" class="pull-right">Last Published: 2018-10-23</li>
+                  <li id="publishDate" class="pull-right">Last Published: 2018-10-24</li>
             </p>
                 </div>
 

http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/constant-values.html
----------------------------------------------------------------------
diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html
index 826687e..1d9399c 100644
--- a/devapidocs/constant-values.html
+++ b/devapidocs/constant-values.html
@@ -3824,7 +3824,7 @@
 <!--   -->
 </a><code>public&nbsp;static&nbsp;final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td><code><a href="org/apache/hadoop/hbase/Version.html#date">date</a></code></td>
-<td class="colLast"><code>"Tue Oct 23 14:46:14 UTC 2018"</code></td>
+<td class="colLast"><code>"Wed Oct 24 14:45:43 UTC 2018"</code></td>
 </tr>
 <tr class="rowColor">
 <td class="colFirst"><a name="org.apache.hadoop.hbase.Version.revision">
@@ -3838,7 +3838,7 @@
 <!--   -->
 </a><code>public&nbsp;static&nbsp;final&nbsp;<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
 <td><code><a href="org/apache/hadoop/hbase/Version.html#srcChecksum">srcChecksum</a></code></td>
-<td class="colLast"><code>"45eea89e2b35169f4606f1b3d8b0a2ee"</code></td>
+<td class="colLast"><code>"1d6019b1fcb1bac7a8be7ca4d582a50d"</code></td>
 </tr>
 <tr class="rowColor">
 <td class="colFirst"><a name="org.apache.hadoop.hbase.Version.url">


[05/51] [partial] hbase-site git commit: Published site at 3fe8649b2c9ba1271c25e8f476548907e4c7a90d.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/8f09a71d/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
index 9b964f6..98ef11a 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.SyncMetrics.html
@@ -105,7 +105,7 @@
 <span class="sourceLineNo">097</span> * will first be initialized to the oldest file's tracker(which is stored in the trailer), using the<a name="line.97"></a>
 <span class="sourceLineNo">098</span> * method {@link ProcedureStoreTracker#resetTo(ProcedureStoreTracker, boolean)}, and then merge it<a name="line.98"></a>
 <span class="sourceLineNo">099</span> * with the tracker of every newer wal files, using the<a name="line.99"></a>
-<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker, boolean)}.<a name="line.100"></a>
+<span class="sourceLineNo">100</span> * {@link ProcedureStoreTracker#setDeletedIfModifiedInBoth(ProcedureStoreTracker)}.<a name="line.100"></a>
 <span class="sourceLineNo">101</span> * If we find out<a name="line.101"></a>
 <span class="sourceLineNo">102</span> * that all the modified procedures for the oldest wal file are modified or deleted in newer wal<a name="line.102"></a>
 <span class="sourceLineNo">103</span> * files, then we can delete it. This is because that, every time we call<a name="line.103"></a>
@@ -1181,244 +1181,243 @@
 <span class="sourceLineNo">1173</span>    }<a name="line.1173"></a>
 <span class="sourceLineNo">1174</span><a name="line.1174"></a>
 <span class="sourceLineNo">1175</span>    // compute the holding tracker.<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span>    //  - the first WAL is used for the 'updates'<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span>    //  - the global tracker is passed in first to decide which procedures are not<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span>    //    exist anymore, so we can mark them as deleted in holdingCleanupTracker.<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span>    //    Only global tracker have the whole picture here.<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span>    //  - the other WALs are scanned to remove procs already updated in a newer wal.<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span>    //    If it is updated in a newer wal, we can mark it as delelted in holdingCleanupTracker<a name="line.1181"></a>
-<span class="sourceLineNo">1182</span>    //    But, we can not delete it if it was shown deleted in the newer wal, as said<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span>    //    above.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span>    // TODO: exit early if holdingCleanupTracker.isEmpty()<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span>    //Passing in the global tracker, we can delete the procedures not in the global<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span>    //tracker, because they are deleted in the later logs<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span>    holdingCleanupTracker.setDeletedIfModifiedInBoth(storeTracker, true);<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span>    for (int i = 1, size = logs.size() - 1; i &lt; size; ++i) {<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span>      // Set deleteIfNotExists to false since a single log's tracker is passed in.<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span>      // Since a specific procedure may not show up in the log at all(not executed or<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span>      // updated during the time), we can not delete the procedure just because this log<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span>      // don't have the info of the procedure. We can delete the procedure only if<a name="line.1193"></a>
-<span class="sourceLineNo">1194</span>      // in this log's tracker, it was cleanly showed that the procedure is modified or deleted<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span>      // in the corresponding BitSetNode.<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(logs.get(i).getTracker(), false);<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span>    }<a name="line.1197"></a>
-<span class="sourceLineNo">1198</span>  }<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span><a name="line.1199"></a>
-<span class="sourceLineNo">1200</span>  /**<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span>   */<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span>    if (logs.size() &lt;= 1) {<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span>      return;<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span>    }<a name="line.1206"></a>
-<span class="sourceLineNo">1207</span><a name="line.1207"></a>
-<span class="sourceLineNo">1208</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span><a name="line.1209"></a>
-<span class="sourceLineNo">1210</span>    boolean removed = false;<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span>    while (logs.size() &gt; 1) {<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span>        break;<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span>      }<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span>      removeLogFile(log, walArchiveDir);<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span>      removed = true;<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span>    }<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span><a name="line.1219"></a>
-<span class="sourceLineNo">1220</span>    if (removed) {<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span>      buildHoldingCleanupTracker();<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span>    }<a name="line.1222"></a>
-<span class="sourceLineNo">1223</span>  }<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span><a name="line.1224"></a>
-<span class="sourceLineNo">1225</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span>    try {<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span>      LOG.trace("Removing log={}", log);<a name="line.1227"></a>
-<span class="sourceLineNo">1228</span>      log.removeFile(walArchiveDir);<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span>      logs.remove(log);<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span>    } catch (IOException e) {<a name="line.1232"></a>
-<span class="sourceLineNo">1233</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span>      return false;<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span>    }<a name="line.1235"></a>
-<span class="sourceLineNo">1236</span>    return true;<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span>  }<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span><a name="line.1238"></a>
-<span class="sourceLineNo">1239</span>  // ==========================================================================<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span>  //  FileSystem Log Files helpers<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span>  // ==========================================================================<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span>  public Path getWALDir() {<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span>    return this.walDir;<a name="line.1243"></a>
-<span class="sourceLineNo">1244</span>  }<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span><a name="line.1245"></a>
-<span class="sourceLineNo">1246</span>  @VisibleForTesting<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span>  Path getWalArchiveDir() {<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span>    return this.walArchiveDir;<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span>  }<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span><a name="line.1250"></a>
-<span class="sourceLineNo">1251</span>  public FileSystem getFileSystem() {<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span>    return this.fs;<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span>  }<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span><a name="line.1254"></a>
-<span class="sourceLineNo">1255</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span>  }<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span><a name="line.1258"></a>
-<span class="sourceLineNo">1259</span>  private static long getLogIdFromName(final String name) {<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span>    int end = name.lastIndexOf(".log");<a name="line.1260"></a>
-<span class="sourceLineNo">1261</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span>    return Long.parseLong(name.substring(start, end));<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span>  }<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span><a name="line.1264"></a>
-<span class="sourceLineNo">1265</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1265"></a>
-<span class="sourceLineNo">1266</span>    @Override<a name="line.1266"></a>
-<span class="sourceLineNo">1267</span>    public boolean accept(Path path) {<a name="line.1267"></a>
-<span class="sourceLineNo">1268</span>      String name = path.getName();<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span>    }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span>  };<a name="line.1271"></a>
-<span class="sourceLineNo">1272</span><a name="line.1272"></a>
-<span class="sourceLineNo">1273</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span>    @Override<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span>      return Long.compare(aId, bId);<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span>    }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span>  };<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span><a name="line.1282"></a>
-<span class="sourceLineNo">1283</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1283"></a>
-<span class="sourceLineNo">1284</span>    try {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span>      return files;<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span>    } catch (FileNotFoundException e) {<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span>      return null;<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span>    }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span>  }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span><a name="line.1293"></a>
-<span class="sourceLineNo">1294</span>  /**<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1295"></a>
-<span class="sourceLineNo">1296</span>   * the file set by log id.<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span>   * @return Max-LogID of the specified log file set<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span>   */<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span>      return 0L;<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span>    }<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span>  }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span><a name="line.1305"></a>
-<span class="sourceLineNo">1306</span>  /**<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1307"></a>
-<span class="sourceLineNo">1308</span>   * the file set by log id.<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span>   * @return Max-LogID of the specified log file set<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span>   */<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span>      return 0L;<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span>    }<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span>    long maxLogId = 0;<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span>      final Path logPath = logFiles[i].getPath();<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span>      if (!isRunning()) {<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span>        throw new IOException("wal aborting");<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span>      }<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span><a name="line.1322"></a>
-<span class="sourceLineNo">1323</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span>      if (log != null) {<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span>        this.logs.add(log);<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span>      }<a name="line.1327"></a>
-<span class="sourceLineNo">1328</span>    }<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span>    initTrackerFromOldLogs();<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span>    return maxLogId;<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span>  }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span>  /**<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span>   */<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span>  private void initTrackerFromOldLogs() {<a name="line.1337"></a>
-<span class="sourceLineNo">1338</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span>      return;<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span>    }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span>    ProcedureWALFile log = logs.getLast();<a name="line.1341"></a>
-<span class="sourceLineNo">1342</span>    if (!log.getTracker().isPartial()) {<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span>      storeTracker.resetTo(log.getTracker());<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span>    } else {<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span>      storeTracker.reset();<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span>      storeTracker.setPartialFlag(true);<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span>    }<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span>  }<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span><a name="line.1349"></a>
-<span class="sourceLineNo">1350</span>  /**<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span>   * Loads given log file and it's tracker.<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span>   */<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span>      throws IOException {<a name="line.1354"></a>
-<span class="sourceLineNo">1355</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1355"></a>
-<span class="sourceLineNo">1356</span>    if (logFile.getLen() == 0) {<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1357"></a>
-<span class="sourceLineNo">1358</span>      log.removeFile(walArchiveDir);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span>      return null;<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span>    }<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span>    try {<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span>      log.open();<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span>      log.removeFile(walArchiveDir);<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span>      return null;<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span>    } catch (IOException e) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span>      LOG.error(msg, e);<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span>      throw new IOException(msg, e);<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span>    }<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span><a name="line.1373"></a>
-<span class="sourceLineNo">1374</span>    try {<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span>      log.readTracker();<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span>    } catch (IOException e) {<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span>      log.getTracker().reset();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span>      log.getTracker().setPartialFlag(true);<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span>    }<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span><a name="line.1381"></a>
-<span class="sourceLineNo">1382</span>    log.close();<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span>    return log;<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span>  }<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span><a name="line.1385"></a>
-<span class="sourceLineNo">1386</span>  /**<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span>   * For testing parse and profiling.<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span>   */<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span>  public static void main(String [] args) throws IOException {<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span>    if (args == null || args.length != 1) {<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span>      System.exit(-1);<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span>    }<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span>        @Override<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span>          // no-op<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span>        }<a name="line.1403"></a>
-<span class="sourceLineNo">1404</span>      });<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span>    try {<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span>      store.start(16);<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span>      pe.init(1, true);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span>    } finally {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span>      store.stop(true);<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span>    }<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span>  }<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span>}<a name="line.1413"></a>
+<span class="sourceLineNo">1176</span>    // - the first WAL is used for the 'updates'<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span>    // - the global tracker will be used to determine whether a procedure has been deleted<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span>    // - other trackers will be used to determine whether a procedure has been updated, as a deleted<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span>    // procedure can always be detected by checking the global tracker, we can save the deleted<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span>    // checks when applying other trackers<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span>    holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true);<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span>    holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span>    // the logs is a linked list, so avoid calling get(index) on it.<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span>    Iterator&lt;ProcedureWALFile&gt; iter = logs.iterator();<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span>    // skip the tracker for the first file when creating the iterator.<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span>    iter.next();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span>    ProcedureStoreTracker tracker = iter.next().getTracker();<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span>    // testing iter.hasNext after calling iter.next to skip applying the tracker for last file,<a name="line.1188"></a>
+<span class="sourceLineNo">1189</span>    // which is just the storeTracker above.<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span>    while (iter.hasNext()) {<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span>      holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker);<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span>      if (holdingCleanupTracker.isEmpty()) {<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span>        break;<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span>      }<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span>      iter.next();<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span>    }<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span>  }<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span><a name="line.1198"></a>
+<span class="sourceLineNo">1199</span>  /**<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span>   * Remove all logs with logId &lt;= {@code lastLogId}.<a name="line.1200"></a>
+<span class="sourceLineNo">1201</span>   */<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span>  private void removeAllLogs(long lastLogId, String why) {<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span>    if (logs.size() &lt;= 1) {<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span>      return;<a name="line.1204"></a>
+<span class="sourceLineNo">1205</span>    }<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span><a name="line.1206"></a>
+<span class="sourceLineNo">1207</span>    LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why);<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span><a name="line.1208"></a>
+<span class="sourceLineNo">1209</span>    boolean removed = false;<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span>    while (logs.size() &gt; 1) {<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span>      ProcedureWALFile log = logs.getFirst();<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span>      if (lastLogId &lt; log.getLogId()) {<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span>        break;<a name="line.1213"></a>
+<span class="sourceLineNo">1214</span>      }<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span>      removeLogFile(log, walArchiveDir);<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span>      removed = true;<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span>    }<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span><a name="line.1218"></a>
+<span class="sourceLineNo">1219</span>    if (removed) {<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span>      buildHoldingCleanupTracker();<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span>    }<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span>  }<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span><a name="line.1223"></a>
+<span class="sourceLineNo">1224</span>  private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) {<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span>    try {<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span>      LOG.trace("Removing log={}", log);<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span>      log.removeFile(walArchiveDir);<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span>      logs.remove(log);<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span>      LOG.debug("Removed log={}, activeLogs={}", log, logs);<a name="line.1229"></a>
+<span class="sourceLineNo">1230</span>      assert logs.size() &gt; 0 : "expected at least one log";<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span>    } catch (IOException e) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span>      LOG.error("Unable to remove log: " + log, e);<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span>      return false;<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span>    }<a name="line.1234"></a>
+<span class="sourceLineNo">1235</span>    return true;<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span>  }<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span><a name="line.1237"></a>
+<span class="sourceLineNo">1238</span>  // ==========================================================================<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span>  //  FileSystem Log Files helpers<a name="line.1239"></a>
+<span class="sourceLineNo">1240</span>  // ==========================================================================<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span>  public Path getWALDir() {<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span>    return this.walDir;<a name="line.1242"></a>
+<span class="sourceLineNo">1243</span>  }<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span><a name="line.1244"></a>
+<span class="sourceLineNo">1245</span>  @VisibleForTesting<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span>  Path getWalArchiveDir() {<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span>    return this.walArchiveDir;<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span>  }<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span><a name="line.1249"></a>
+<span class="sourceLineNo">1250</span>  public FileSystem getFileSystem() {<a name="line.1250"></a>
+<span class="sourceLineNo">1251</span>    return this.fs;<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span>  }<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span><a name="line.1253"></a>
+<span class="sourceLineNo">1254</span>  protected Path getLogFilePath(final long logId) throws IOException {<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span>    return new Path(walDir, String.format(LOG_PREFIX + "%020d.log", logId));<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span>  }<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span><a name="line.1257"></a>
+<span class="sourceLineNo">1258</span>  private static long getLogIdFromName(final String name) {<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span>    int end = name.lastIndexOf(".log");<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span>    int start = name.lastIndexOf('-') + 1;<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span>    return Long.parseLong(name.substring(start, end));<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span>  }<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span><a name="line.1263"></a>
+<span class="sourceLineNo">1264</span>  private static final PathFilter WALS_PATH_FILTER = new PathFilter() {<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span>    @Override<a name="line.1265"></a>
+<span class="sourceLineNo">1266</span>    public boolean accept(Path path) {<a name="line.1266"></a>
+<span class="sourceLineNo">1267</span>      String name = path.getName();<a name="line.1267"></a>
+<span class="sourceLineNo">1268</span>      return name.startsWith(LOG_PREFIX) &amp;&amp; name.endsWith(".log");<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span>    }<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span>  };<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span><a name="line.1271"></a>
+<span class="sourceLineNo">1272</span>  private static final Comparator&lt;FileStatus&gt; FILE_STATUS_ID_COMPARATOR =<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span>      new Comparator&lt;FileStatus&gt;() {<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span>    @Override<a name="line.1274"></a>
+<span class="sourceLineNo">1275</span>    public int compare(FileStatus a, FileStatus b) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span>      final long aId = getLogIdFromName(a.getPath().getName());<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span>      final long bId = getLogIdFromName(b.getPath().getName());<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span>      return Long.compare(aId, bId);<a name="line.1278"></a>
+<span class="sourceLineNo">1279</span>    }<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span>  };<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span><a name="line.1281"></a>
+<span class="sourceLineNo">1282</span>  private FileStatus[] getLogFiles() throws IOException {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span>    try {<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span>      FileStatus[] files = fs.listStatus(walDir, WALS_PATH_FILTER);<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span>      Arrays.sort(files, FILE_STATUS_ID_COMPARATOR);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span>      return files;<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span>    } catch (FileNotFoundException e) {<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span>      LOG.warn("Log directory not found: " + e.getMessage());<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span>      return null;<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span>    }<a name="line.1290"></a>
+<span class="sourceLineNo">1291</span>  }<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span><a name="line.1292"></a>
+<span class="sourceLineNo">1293</span>  /**<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span>   * the file set by log id.<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span>   * @return Max-LogID of the specified log file set<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span>   */<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span>  private static long getMaxLogId(FileStatus[] logFiles) {<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span>      return 0L;<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span>    }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span>    return getLogIdFromName(logFiles[logFiles.length - 1].getPath().getName());<a name="line.1302"></a>
+<span class="sourceLineNo">1303</span>  }<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span><a name="line.1304"></a>
+<span class="sourceLineNo">1305</span>  /**<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span>   * Make sure that the file set are gotten by calling {@link #getLogFiles()}, where we will sort<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span>   * the file set by log id.<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span>   * @return Max-LogID of the specified log file set<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span>   */<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span>  private long initOldLogs(FileStatus[] logFiles) throws IOException {<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span>    if (logFiles == null || logFiles.length == 0) {<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span>      return 0L;<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span>    }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span>    long maxLogId = 0;<a name="line.1314"></a>
+<span class="sourceLineNo">1315</span>    for (int i = 0; i &lt; logFiles.length; ++i) {<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span>      final Path logPath = logFiles[i].getPath();<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span>      leaseRecovery.recoverFileLease(fs, logPath);<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span>      if (!isRunning()) {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span>        throw new IOException("wal aborting");<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span>      }<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span><a name="line.1321"></a>
+<span class="sourceLineNo">1322</span>      maxLogId = Math.max(maxLogId, getLogIdFromName(logPath.getName()));<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span>      ProcedureWALFile log = initOldLog(logFiles[i], this.walArchiveDir);<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span>      if (log != null) {<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span>        this.logs.add(log);<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span>      }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span>    }<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span>    initTrackerFromOldLogs();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span>    return maxLogId;<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span>  }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span><a name="line.1331"></a>
+<span class="sourceLineNo">1332</span>  /**<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span>   * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span>   * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log.<a name="line.1334"></a>
+<span class="sourceLineNo">1335</span>   */<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span>  private void initTrackerFromOldLogs() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span>    if (logs.isEmpty() || !isRunning()) {<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span>      return;<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span>    }<a name="line.1339"></a>
+<span class="sourceLineNo">1340</span>    ProcedureWALFile log = logs.getLast();<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span>    if (!log.getTracker().isPartial()) {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span>      storeTracker.resetTo(log.getTracker());<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span>    } else {<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span>      storeTracker.reset();<a name="line.1344"></a>
+<span class="sourceLineNo">1345</span>      storeTracker.setPartialFlag(true);<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span>    }<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span>  }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span>  /**<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span>   * Loads given log file and it's tracker.<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span>   */<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span>  private ProcedureWALFile initOldLog(final FileStatus logFile, final Path walArchiveDir)<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span>      throws IOException {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span>    final ProcedureWALFile log = new ProcedureWALFile(fs, logFile);<a name="line.1354"></a>
+<span class="sourceLineNo">1355</span>    if (logFile.getLen() == 0) {<a name="line.1355"></a>
+<span class="sourceLineNo">1356</span>      LOG.warn("Remove uninitialized log: {}", logFile);<a name="line.1356"></a>
+<span class="sourceLineNo">1357</span>      log.removeFile(walArchiveDir);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span>      return null;<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span>    }<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span>    LOG.debug("Opening Pv2 {}", logFile);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span>    try {<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span>      log.open();<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span>    } catch (ProcedureWALFormat.InvalidWALDataException e) {<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span>      LOG.warn("Remove uninitialized log: {}", logFile, e);<a name="line.1364"></a>
+<span class="sourceLineNo">1365</span>      log.removeFile(walArchiveDir);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span>      return null;<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span>    } catch (IOException e) {<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span>      String msg = "Unable to read state log: " + logFile;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span>      LOG.error(msg, e);<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span>      throw new IOException(msg, e);<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span>    }<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span><a name="line.1372"></a>
+<span class="sourceLineNo">1373</span>    try {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span>      log.readTracker();<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span>    } catch (IOException e) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span>      log.getTracker().reset();<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span>      log.getTracker().setPartialFlag(true);<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span>      LOG.warn("Unable to read tracker for {}", log, e);<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span>    }<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span><a name="line.1380"></a>
+<span class="sourceLineNo">1381</span>    log.close();<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span>    return log;<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span>  }<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span><a name="line.1384"></a>
+<span class="sourceLineNo">1385</span>  /**<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span>   * Parses a directory of WALs building up ProcedureState.<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span>   * For testing parse and profiling.<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span>   * @param args Include pointer to directory of WAL files for a store instance to parse &amp; load.<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span>   */<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span>  public static void main(String [] args) throws IOException {<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span>    Configuration conf = HBaseConfiguration.create();<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span>    if (args == null || args.length != 1) {<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span>      System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span>      System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span>      System.exit(-1);<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span>    }<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span>    WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span>      new WALProcedureStore.LeaseRecovery() {<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span>        @Override<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span>        public void recoverFileLease(FileSystem fs, Path path) throws IOException {<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span>          // no-op<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span>        }<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span>      });<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span>    try {<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span>      store.start(16);<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span>      ProcedureExecutor&lt;?&gt; pe = new ProcedureExecutor&lt;&gt;(conf, new Object()/*Pass anything*/, store);<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span>      pe.init(1, true);<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span>    } finally {<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span>      store.stop(true);<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span>    }<a name="line.1410"></a>
+<span class="sourceLineNo">1411</span>  }<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span>}<a name="line.1412"></a>