You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2026/09/04 00:00:36 UTC
(hbase-site) branch asf-site updated: Published site at 5432392fe1fc395213f0f4da65782181b26032e6.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hbase-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 9fb8b40fcf3 Published site at 5432392fe1fc395213f0f4da65782181b26032e6.
9fb8b40fcf3 is described below
commit 9fb8b40fcf3adb451fd95f832759a0d13316da76
Author: HBase <de...@hbase.apache.org>
AuthorDate: Thu Sep 3 23:55:58 2026 +0000
Published site at 5432392fe1fc395213f0f4da65782181b26032e6.
---
.../hadoop/hbase/mob/MobFileCleanupUtil.html | 457 +++++++++++----------
.../hadoop/hbase/regionserver/wal/FSHLog.html | 44 +-
books/apache-hbase-reference-guide-dark-mode.pdf | Bin 38775819 -> 38775817 bytes
books/apache-hbase-reference-guide.pdf | Bin 38665189 -> 38665189 bytes
checkstyle-aggregate.html | 2 +-
dependency-convergence.html | 2 +-
dependency-info.html | 2 +-
dependency-management.html | 2 +-
.../hadoop/hbase/mob/MobFileCleanupUtil.html | 12 +-
.../hadoop/hbase/regionserver/wal/FSHLog.html | 10 +-
.../hadoop/hbase/mob/MobFileCleanupUtil.html | 457 +++++++++++----------
.../hbase/regionserver/wal/FSHLog.SyncRequest.html | 44 +-
.../hbase/regionserver/wal/FSHLog.SyncRunner.html | 44 +-
.../hadoop/hbase/regionserver/wal/FSHLog.html | 44 +-
hbase-website/dependency-convergence.html | 2 +-
hbase-website/dependency-info.html | 2 +-
hbase-website/dependency-management.html | 2 +-
hbase-website/index.html | 2 +-
hbase-website/issue-management.html | 2 +-
hbase-website/licenses.html | 2 +-
hbase-website/mailing-lists.html | 2 +-
hbase-website/plugin-management.html | 2 +-
hbase-website/plugins.html | 2 +-
hbase-website/project-info.html | 2 +-
hbase-website/scm.html | 2 +-
hbase-website/summary.html | 2 +-
hbase-website/team.html | 2 +-
issue-management.html | 2 +-
licenses.html | 2 +-
mailing-lists.html | 2 +-
plugin-management.html | 2 +-
plugins.html | 2 +-
project-info.html | 2 +-
project-reports.html | 2 +-
scm.html | 2 +-
summary.html | 2 +-
team.html | 2 +-
testapidocs/index-all.html | 5 +
testapidocs/member-search-index.js | 2 +-
.../hadoop/hbase/regionserver/wal/TestFSHLog.html | 430 ++++++++++---------
testdevapidocs/index-all.html | 5 +
testdevapidocs/member-search-index.js | 2 +-
.../hadoop/hbase/regionserver/wal/TestFSHLog.html | 49 ++-
.../hadoop/hbase/regionserver/wal/TestFSHLog.html | 430 ++++++++++---------
44 files changed, 1121 insertions(+), 968 deletions(-)
diff --git a/apidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html b/apidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
index f247284b008..d9c748162fa 100644
--- a/apidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
+++ b/apidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
@@ -33,234 +33,235 @@
<span class="source-line-no">020</span><span id="line-20">import java.io.FileNotFoundException;</span>
<span class="source-line-no">021</span><span id="line-21">import java.io.IOException;</span>
<span class="source-line-no">022</span><span id="line-22">import java.util.ArrayList;</span>
-<span class="source-line-no">023</span><span id="line-23">import java.util.HashSet;</span>
-<span class="source-line-no">024</span><span id="line-24">import java.util.List;</span>
-<span class="source-line-no">025</span><span id="line-25">import java.util.Set;</span>
-<span class="source-line-no">026</span><span id="line-26">import org.apache.hadoop.conf.Configuration;</span>
-<span class="source-line-no">027</span><span id="line-27">import org.apache.hadoop.fs.FileSystem;</span>
-<span class="source-line-no">028</span><span id="line-28">import org.apache.hadoop.fs.LocatedFileStatus;</span>
-<span class="source-line-no">029</span><span id="line-29">import org.apache.hadoop.fs.Path;</span>
-<span class="source-line-no">030</span><span id="line-30">import org.apache.hadoop.fs.RemoteIterator;</span>
-<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.hbase.TableName;</span>
-<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.hbase.backup.HFileArchiver;</span>
-<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.hbase.client.Admin;</span>
-<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span>
-<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
-<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span>
-<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.regionserver.BloomType;</span>
-<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;</span>
-<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.regionserver.HStoreFile;</span>
-<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span>
-<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;</span>
-<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;</span>
-<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.util.Bytes;</span>
-<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
-<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span>
-<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.util.FSUtils;</span>
-<span class="source-line-no">047</span><span id="line-47">import org.apache.yetus.audience.InterfaceAudience;</span>
-<span class="source-line-no">048</span><span id="line-48">import org.slf4j.Logger;</span>
-<span class="source-line-no">049</span><span id="line-49">import org.slf4j.LoggerFactory;</span>
-<span class="source-line-no">050</span><span id="line-50"></span>
-<span class="source-line-no">051</span><span id="line-51">import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap;</span>
-<span class="source-line-no">052</span><span id="line-52"></span>
-<span class="source-line-no">053</span><span id="line-53">@InterfaceAudience.Private</span>
-<span class="source-line-no">054</span><span id="line-54">public final class MobFileCleanupUtil {</span>
-<span class="source-line-no">055</span><span id="line-55"></span>
-<span class="source-line-no">056</span><span id="line-56"> private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class);</span>
-<span class="source-line-no">057</span><span id="line-57"></span>
-<span class="source-line-no">058</span><span id="line-58"> private MobFileCleanupUtil() {</span>
-<span class="source-line-no">059</span><span id="line-59"> }</span>
-<span class="source-line-no">060</span><span id="line-60"></span>
-<span class="source-line-no">061</span><span id="line-61"> /**</span>
-<span class="source-line-no">062</span><span id="line-62"> * Performs housekeeping file cleaning (called by MOB Cleaner chore)</span>
-<span class="source-line-no">063</span><span id="line-63"> * @param conf configuration</span>
-<span class="source-line-no">064</span><span id="line-64"> * @param table table name</span>
-<span class="source-line-no">065</span><span id="line-65"> * @throws IOException exception</span>
-<span class="source-line-no">066</span><span id="line-66"> */</span>
-<span class="source-line-no">067</span><span id="line-67"> public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin)</span>
-<span class="source-line-no">068</span><span id="line-68"> throws IOException {</span>
-<span class="source-line-no">069</span><span id="line-69"> long minAgeToArchive =</span>
-<span class="source-line-no">070</span><span id="line-70"> conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);</span>
-<span class="source-line-no">071</span><span id="line-71"> // We check only those MOB files, which creation time is less</span>
-<span class="source-line-no">072</span><span id="line-72"> // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap</span>
-<span class="source-line-no">073</span><span id="line-73"> // gives us full confidence that all corresponding store files will</span>
-<span class="source-line-no">074</span><span id="line-74"> // exist at the time cleaning procedure begins and will be examined.</span>
-<span class="source-line-no">075</span><span id="line-75"> // So, if MOB file creation time is greater than this maxTimeToArchive,</span>
-<span class="source-line-no">076</span><span id="line-76"> // this will be skipped and won't be archived.</span>
-<span class="source-line-no">077</span><span id="line-77"> long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;</span>
-<span class="source-line-no">078</span><span id="line-78"> TableDescriptor htd = admin.getDescriptor(table);</span>
-<span class="source-line-no">079</span><span id="line-79"> List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);</span>
-<span class="source-line-no">080</span><span id="line-80"> if (list.size() == 0) {</span>
-<span class="source-line-no">081</span><span id="line-81"> LOG.info("Skipping non-MOB table [{}]", table);</span>
-<span class="source-line-no">082</span><span id="line-82"> return;</span>
-<span class="source-line-no">083</span><span id="line-83"> } else {</span>
-<span class="source-line-no">084</span><span id="line-84"> LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",</span>
-<span class="source-line-no">085</span><span id="line-85"> maxCreationTimeToArchive, table);</span>
-<span class="source-line-no">086</span><span id="line-86"> }</span>
-<span class="source-line-no">087</span><span id="line-87"></span>
-<span class="source-line-no">088</span><span id="line-88"> FileSystem fs = FileSystem.get(conf);</span>
-<span class="source-line-no">089</span><span id="line-89"> Set<String> regionNames = new HashSet<>();</span>
-<span class="source-line-no">090</span><span id="line-90"> Path rootDir = CommonFSUtils.getRootDir(conf);</span>
-<span class="source-line-no">091</span><span id="line-91"> Path tableDir = CommonFSUtils.getTableDir(rootDir, table);</span>
-<span class="source-line-no">092</span><span id="line-92"> List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);</span>
-<span class="source-line-no">093</span><span id="line-93"></span>
-<span class="source-line-no">094</span><span id="line-94"> Set<String> allActiveMobFileName = new HashSet<String>();</span>
-<span class="source-line-no">095</span><span id="line-95"> for (Path regionPath : regionDirs) {</span>
-<span class="source-line-no">096</span><span id="line-96"> regionNames.add(regionPath.getName());</span>
-<span class="source-line-no">097</span><span id="line-97"> HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir,</span>
-<span class="source-line-no">098</span><span id="line-98"> HRegionFileSystem.loadRegionInfoFileContent(fs, regionPath));</span>
-<span class="source-line-no">099</span><span id="line-99"> for (ColumnFamilyDescriptor hcd : list) {</span>
-<span class="source-line-no">100</span><span id="line-100"> StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false);</span>
-<span class="source-line-no">101</span><span id="line-101"> String family = hcd.getNameAsString();</span>
-<span class="source-line-no">102</span><span id="line-102"> Path storePath = new Path(regionPath, family);</span>
-<span class="source-line-no">103</span><span id="line-103"> boolean succeed = false;</span>
-<span class="source-line-no">104</span><span id="line-104"> Set<String> regionMobs = new HashSet<String>();</span>
-<span class="source-line-no">105</span><span id="line-105"></span>
-<span class="source-line-no">106</span><span id="line-106"> while (!succeed) {</span>
-<span class="source-line-no">107</span><span id="line-107"> if (!fs.exists(storePath)) {</span>
-<span class="source-line-no">108</span><span id="line-108"> String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore"</span>
-<span class="source-line-no">109</span><span id="line-109"> + " execution, aborting MOB file cleaner chore.", storePath);</span>
-<span class="source-line-no">110</span><span id="line-110"> throw new IOException(errMsg);</span>
-<span class="source-line-no">111</span><span id="line-111"> }</span>
-<span class="source-line-no">112</span><span id="line-112"> List<StoreFileInfo> storeFileInfos = sft.load();</span>
-<span class="source-line-no">113</span><span id="line-113"> LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath);</span>
-<span class="source-line-no">114</span><span id="line-114"> Path currentPath = null;</span>
-<span class="source-line-no">115</span><span id="line-115"> try {</span>
-<span class="source-line-no">116</span><span id="line-116"> for (StoreFileInfo storeFileInfo : storeFileInfos) {</span>
-<span class="source-line-no">117</span><span id="line-117"> Path pp = storeFileInfo.getPath();</span>
-<span class="source-line-no">118</span><span id="line-118"> currentPath = pp;</span>
-<span class="source-line-no">119</span><span id="line-119"> LOG.trace("Store file: {}", pp);</span>
-<span class="source-line-no">120</span><span id="line-120"> HStoreFile sf = null;</span>
-<span class="source-line-no">121</span><span id="line-121"> byte[] mobRefData = null;</span>
-<span class="source-line-no">122</span><span id="line-122"> byte[] bulkloadMarkerData = null;</span>
-<span class="source-line-no">123</span><span id="line-123"> try {</span>
-<span class="source-line-no">124</span><span id="line-124"> sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED);</span>
-<span class="source-line-no">125</span><span id="line-125"> sf.initReader();</span>
-<span class="source-line-no">126</span><span id="line-126"> mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);</span>
-<span class="source-line-no">127</span><span id="line-127"> bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);</span>
-<span class="source-line-no">128</span><span id="line-128"> // close store file to avoid memory leaks</span>
-<span class="source-line-no">129</span><span id="line-129"> sf.closeStoreFile(true);</span>
-<span class="source-line-no">130</span><span id="line-130"> } catch (IOException ex) {</span>
-<span class="source-line-no">131</span><span id="line-131"> // When FileBased SFT is active the store dir can contain corrupted or incomplete</span>
-<span class="source-line-no">132</span><span id="line-132"> // files. So read errors are expected. We just skip these files.</span>
-<span class="source-line-no">133</span><span id="line-133"> if (ex instanceof FileNotFoundException) {</span>
-<span class="source-line-no">134</span><span id="line-134"> throw ex;</span>
-<span class="source-line-no">135</span><span id="line-135"> }</span>
-<span class="source-line-no">136</span><span id="line-136"> LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex);</span>
-<span class="source-line-no">137</span><span id="line-137"> continue;</span>
-<span class="source-line-no">138</span><span id="line-138"> }</span>
-<span class="source-line-no">139</span><span id="line-139"> if (mobRefData == null) {</span>
-<span class="source-line-no">140</span><span id="line-140"> if (bulkloadMarkerData == null) {</span>
-<span class="source-line-no">141</span><span id="line-141"> LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "</span>
-<span class="source-line-no">142</span><span id="line-142"> + "can not proceed until all old files will be MOB-compacted.", pp);</span>
-<span class="source-line-no">143</span><span id="line-143"> return;</span>
-<span class="source-line-no">144</span><span id="line-144"> } else {</span>
-<span class="source-line-no">145</span><span id="line-145"> LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);</span>
-<span class="source-line-no">146</span><span id="line-146"> continue;</span>
-<span class="source-line-no">147</span><span id="line-147"> }</span>
-<span class="source-line-no">148</span><span id="line-148"> }</span>
-<span class="source-line-no">149</span><span id="line-149"> // file may or may not have MOB references, but was created by the distributed</span>
-<span class="source-line-no">150</span><span id="line-150"> // mob compaction code.</span>
-<span class="source-line-no">151</span><span id="line-151"> try {</span>
-<span class="source-line-no">152</span><span id="line-152"> SetMultimap<TableName, String> mobs =</span>
-<span class="source-line-no">153</span><span id="line-153"> MobUtils.deserializeMobFileRefs(mobRefData).build();</span>
-<span class="source-line-no">154</span><span id="line-154"> LOG.debug("Found {} mob references for store={}", mobs.size(), sf);</span>
-<span class="source-line-no">155</span><span id="line-155"> LOG.trace("Specific mob references found for store={} : {}", sf, mobs);</span>
-<span class="source-line-no">156</span><span id="line-156"> regionMobs.addAll(mobs.values());</span>
-<span class="source-line-no">157</span><span id="line-157"> } catch (RuntimeException exception) {</span>
-<span class="source-line-no">158</span><span id="line-158"> throw new IOException("failure getting mob references for hfile " + sf, exception);</span>
-<span class="source-line-no">159</span><span id="line-159"> }</span>
-<span class="source-line-no">160</span><span id="line-160"> }</span>
-<span class="source-line-no">161</span><span id="line-161"> } catch (FileNotFoundException e) {</span>
-<span class="source-line-no">162</span><span id="line-162"> LOG.warn(</span>
-<span class="source-line-no">163</span><span id="line-163"> "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error",</span>
-<span class="source-line-no">164</span><span id="line-164"> currentPath, e);</span>
-<span class="source-line-no">165</span><span id="line-165"> regionMobs.clear();</span>
-<span class="source-line-no">166</span><span id="line-166"> continue;</span>
-<span class="source-line-no">167</span><span id="line-167"> }</span>
-<span class="source-line-no">168</span><span id="line-168"> succeed = true;</span>
-<span class="source-line-no">169</span><span id="line-169"> }</span>
-<span class="source-line-no">170</span><span id="line-170"></span>
-<span class="source-line-no">171</span><span id="line-171"> // Add MOB references for current region/family</span>
-<span class="source-line-no">172</span><span id="line-172"> allActiveMobFileName.addAll(regionMobs);</span>
-<span class="source-line-no">173</span><span id="line-173"> } // END column families</span>
-<span class="source-line-no">174</span><span id="line-174"> } // END regions</span>
-<span class="source-line-no">175</span><span id="line-175"> // Check if number of MOB files too big (over 1M)</span>
-<span class="source-line-no">176</span><span id="line-176"> if (allActiveMobFileName.size() > 1000000) {</span>
-<span class="source-line-no">177</span><span id="line-177"> LOG.warn("Found too many active MOB files: {}, table={}, "</span>
-<span class="source-line-no">178</span><span id="line-178"> + "this may result in high memory pressure.", allActiveMobFileName.size(), table);</span>
-<span class="source-line-no">179</span><span id="line-179"> }</span>
-<span class="source-line-no">180</span><span id="line-180"> LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);</span>
-<span class="source-line-no">181</span><span id="line-181"> allActiveMobFileName.stream().forEach(LOG::trace);</span>
-<span class="source-line-no">182</span><span id="line-182"></span>
-<span class="source-line-no">183</span><span id="line-183"> // Now scan MOB directories and find MOB files with no references to them</span>
-<span class="source-line-no">184</span><span id="line-184"> for (ColumnFamilyDescriptor hcd : list) {</span>
-<span class="source-line-no">185</span><span id="line-185"> checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames,</span>
-<span class="source-line-no">186</span><span id="line-186"> maxCreationTimeToArchive);</span>
-<span class="source-line-no">187</span><span id="line-187"> }</span>
-<span class="source-line-no">188</span><span id="line-188"> }</span>
-<span class="source-line-no">189</span><span id="line-189"></span>
-<span class="source-line-no">190</span><span id="line-190"> private static void checkColumnFamilyDescriptor(Configuration conf, TableName table,</span>
-<span class="source-line-no">191</span><span id="line-191"> FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set<String> regionNames,</span>
-<span class="source-line-no">192</span><span id="line-192"> long maxCreationTimeToArchive) throws IOException {</span>
-<span class="source-line-no">193</span><span id="line-193"> List<Path> toArchive = new ArrayList<Path>();</span>
-<span class="source-line-no">194</span><span id="line-194"> String family = hcd.getNameAsString();</span>
-<span class="source-line-no">195</span><span id="line-195"> Path dir = MobUtils.getMobFamilyPath(conf, table, family);</span>
-<span class="source-line-no">196</span><span id="line-196"> RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);</span>
-<span class="source-line-no">197</span><span id="line-197"> while (rit.hasNext()) {</span>
-<span class="source-line-no">198</span><span id="line-198"> LocatedFileStatus lfs = rit.next();</span>
-<span class="source-line-no">199</span><span id="line-199"> Path p = lfs.getPath();</span>
-<span class="source-line-no">200</span><span id="line-200"> String[] mobParts = p.getName().split("_");</span>
-<span class="source-line-no">201</span><span id="line-201"> String regionName = mobParts[mobParts.length - 1];</span>
-<span class="source-line-no">202</span><span id="line-202"></span>
-<span class="source-line-no">203</span><span id="line-203"> if (!regionNames.contains(regionName)) {</span>
-<span class="source-line-no">204</span><span id="line-204"> // MOB belonged to a region no longer hosted</span>
-<span class="source-line-no">205</span><span id="line-205"> long creationTime = fs.getFileStatus(p).getModificationTime();</span>
-<span class="source-line-no">206</span><span id="line-206"> if (creationTime < maxCreationTimeToArchive) {</span>
-<span class="source-line-no">207</span><span id="line-207"> LOG.trace("Archiving MOB file {} creation time={}", p,</span>
-<span class="source-line-no">208</span><span id="line-208"> (fs.getFileStatus(p).getModificationTime()));</span>
-<span class="source-line-no">209</span><span id="line-209"> toArchive.add(p);</span>
-<span class="source-line-no">210</span><span id="line-210"> } else {</span>
-<span class="source-line-no">211</span><span id="line-211"> LOG.trace("Skipping fresh file: {}. Creation time={}", p,</span>
-<span class="source-line-no">212</span><span id="line-212"> fs.getFileStatus(p).getModificationTime());</span>
-<span class="source-line-no">213</span><span id="line-213"> }</span>
-<span class="source-line-no">214</span><span id="line-214"> } else {</span>
-<span class="source-line-no">215</span><span id="line-215"> LOG.trace("Keeping MOB file with existing region: {}", p);</span>
-<span class="source-line-no">216</span><span id="line-216"> }</span>
-<span class="source-line-no">217</span><span id="line-217"> }</span>
-<span class="source-line-no">218</span><span id="line-218"> LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(),</span>
-<span class="source-line-no">219</span><span id="line-219"> table, family);</span>
-<span class="source-line-no">220</span><span id="line-220"> archiveMobFiles(conf, table, admin, family.getBytes(), toArchive);</span>
-<span class="source-line-no">221</span><span id="line-221"> LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);</span>
-<span class="source-line-no">222</span><span id="line-222"> }</span>
-<span class="source-line-no">223</span><span id="line-223"></span>
-<span class="source-line-no">224</span><span id="line-224"> /**</span>
-<span class="source-line-no">225</span><span id="line-225"> * Archives the mob files.</span>
-<span class="source-line-no">226</span><span id="line-226"> * @param conf The current configuration.</span>
-<span class="source-line-no">227</span><span id="line-227"> * @param tableName The table name.</span>
-<span class="source-line-no">228</span><span id="line-228"> * @param family The name of the column family.</span>
-<span class="source-line-no">229</span><span id="line-229"> * @param storeFiles The files to be archived.</span>
-<span class="source-line-no">230</span><span id="line-230"> * @throws IOException exception</span>
-<span class="source-line-no">231</span><span id="line-231"> */</span>
-<span class="source-line-no">232</span><span id="line-232"> private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin,</span>
-<span class="source-line-no">233</span><span id="line-233"> byte[] family, List<Path> storeFiles) throws IOException {</span>
-<span class="source-line-no">234</span><span id="line-234"></span>
-<span class="source-line-no">235</span><span id="line-235"> if (storeFiles.size() == 0) {</span>
-<span class="source-line-no">236</span><span id="line-236"> // nothing to remove</span>
-<span class="source-line-no">237</span><span id="line-237"> LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName,</span>
-<span class="source-line-no">238</span><span id="line-238"> Bytes.toString(family));</span>
-<span class="source-line-no">239</span><span id="line-239"> return;</span>
-<span class="source-line-no">240</span><span id="line-240"> }</span>
-<span class="source-line-no">241</span><span id="line-241"> Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);</span>
-<span class="source-line-no">242</span><span id="line-242"> FileSystem fs = storeFiles.get(0).getFileSystem(conf);</span>
-<span class="source-line-no">243</span><span id="line-243"></span>
-<span class="source-line-no">244</span><span id="line-244"> for (Path p : storeFiles) {</span>
-<span class="source-line-no">245</span><span id="line-245"> LOG.debug("MOB Cleaner is archiving: {}", p);</span>
-<span class="source-line-no">246</span><span id="line-246"> HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,</span>
-<span class="source-line-no">247</span><span id="line-247"> family, p);</span>
-<span class="source-line-no">248</span><span id="line-248"> }</span>
-<span class="source-line-no">249</span><span id="line-249"> }</span>
-<span class="source-line-no">250</span><span id="line-250">}</span>
+<span class="source-line-no">023</span><span id="line-23">import java.util.Date;</span>
+<span class="source-line-no">024</span><span id="line-24">import java.util.HashSet;</span>
+<span class="source-line-no">025</span><span id="line-25">import java.util.List;</span>
+<span class="source-line-no">026</span><span id="line-26">import java.util.Set;</span>
+<span class="source-line-no">027</span><span id="line-27">import org.apache.hadoop.conf.Configuration;</span>
+<span class="source-line-no">028</span><span id="line-28">import org.apache.hadoop.fs.FileSystem;</span>
+<span class="source-line-no">029</span><span id="line-29">import org.apache.hadoop.fs.LocatedFileStatus;</span>
+<span class="source-line-no">030</span><span id="line-30">import org.apache.hadoop.fs.Path;</span>
+<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.fs.RemoteIterator;</span>
+<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.hbase.TableName;</span>
+<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.hbase.backup.HFileArchiver;</span>
+<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.client.Admin;</span>
+<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span>
+<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
+<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span>
+<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.regionserver.BloomType;</span>
+<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;</span>
+<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.regionserver.HStoreFile;</span>
+<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span>
+<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;</span>
+<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;</span>
+<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.util.Bytes;</span>
+<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
+<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span>
+<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.util.FSUtils;</span>
+<span class="source-line-no">048</span><span id="line-48">import org.apache.yetus.audience.InterfaceAudience;</span>
+<span class="source-line-no">049</span><span id="line-49">import org.slf4j.Logger;</span>
+<span class="source-line-no">050</span><span id="line-50">import org.slf4j.LoggerFactory;</span>
+<span class="source-line-no">051</span><span id="line-51"></span>
+<span class="source-line-no">052</span><span id="line-52">import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap;</span>
+<span class="source-line-no">053</span><span id="line-53"></span>
+<span class="source-line-no">054</span><span id="line-54">@InterfaceAudience.Private</span>
+<span class="source-line-no">055</span><span id="line-55">public final class MobFileCleanupUtil {</span>
+<span class="source-line-no">056</span><span id="line-56"></span>
+<span class="source-line-no">057</span><span id="line-57"> private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class);</span>
+<span class="source-line-no">058</span><span id="line-58"></span>
+<span class="source-line-no">059</span><span id="line-59"> private MobFileCleanupUtil() {</span>
+<span class="source-line-no">060</span><span id="line-60"> }</span>
+<span class="source-line-no">061</span><span id="line-61"></span>
+<span class="source-line-no">062</span><span id="line-62"> /**</span>
+<span class="source-line-no">063</span><span id="line-63"> * Performs housekeeping file cleaning (called by MOB Cleaner chore)</span>
+<span class="source-line-no">064</span><span id="line-64"> * @param conf configuration</span>
+<span class="source-line-no">065</span><span id="line-65"> * @param table table name</span>
+<span class="source-line-no">066</span><span id="line-66"> * @throws IOException exception</span>
+<span class="source-line-no">067</span><span id="line-67"> */</span>
+<span class="source-line-no">068</span><span id="line-68"> public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin)</span>
+<span class="source-line-no">069</span><span id="line-69"> throws IOException {</span>
+<span class="source-line-no">070</span><span id="line-70"> long minAgeToArchive =</span>
+<span class="source-line-no">071</span><span id="line-71"> conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);</span>
+<span class="source-line-no">072</span><span id="line-72"> // We check only those MOB files, which creation time is less</span>
+<span class="source-line-no">073</span><span id="line-73"> // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap</span>
+<span class="source-line-no">074</span><span id="line-74"> // gives us full confidence that all corresponding store files will</span>
+<span class="source-line-no">075</span><span id="line-75"> // exist at the time cleaning procedure begins and will be examined.</span>
+<span class="source-line-no">076</span><span id="line-76"> // So, if MOB file creation time is greater than this maxTimeToArchive,</span>
+<span class="source-line-no">077</span><span id="line-77"> // this will be skipped and won't be archived.</span>
+<span class="source-line-no">078</span><span id="line-78"> long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;</span>
+<span class="source-line-no">079</span><span id="line-79"> TableDescriptor htd = admin.getDescriptor(table);</span>
+<span class="source-line-no">080</span><span id="line-80"> List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);</span>
+<span class="source-line-no">081</span><span id="line-81"> if (list.size() == 0) {</span>
+<span class="source-line-no">082</span><span id="line-82"> LOG.info("Skipping non-MOB table [{}]", table);</span>
+<span class="source-line-no">083</span><span id="line-83"> return;</span>
+<span class="source-line-no">084</span><span id="line-84"> } else {</span>
+<span class="source-line-no">085</span><span id="line-85"> LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",</span>
+<span class="source-line-no">086</span><span id="line-86"> new Date(maxCreationTimeToArchive), table);</span>
+<span class="source-line-no">087</span><span id="line-87"> }</span>
+<span class="source-line-no">088</span><span id="line-88"></span>
+<span class="source-line-no">089</span><span id="line-89"> FileSystem fs = FileSystem.get(conf);</span>
+<span class="source-line-no">090</span><span id="line-90"> Set<String> regionNames = new HashSet<>();</span>
+<span class="source-line-no">091</span><span id="line-91"> Path rootDir = CommonFSUtils.getRootDir(conf);</span>
+<span class="source-line-no">092</span><span id="line-92"> Path tableDir = CommonFSUtils.getTableDir(rootDir, table);</span>
+<span class="source-line-no">093</span><span id="line-93"> List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);</span>
+<span class="source-line-no">094</span><span id="line-94"></span>
+<span class="source-line-no">095</span><span id="line-95"> Set<String> allActiveMobFileName = new HashSet<String>();</span>
+<span class="source-line-no">096</span><span id="line-96"> for (Path regionPath : regionDirs) {</span>
+<span class="source-line-no">097</span><span id="line-97"> regionNames.add(regionPath.getName());</span>
+<span class="source-line-no">098</span><span id="line-98"> HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir,</span>
+<span class="source-line-no">099</span><span id="line-99"> HRegionFileSystem.loadRegionInfoFileContent(fs, regionPath));</span>
+<span class="source-line-no">100</span><span id="line-100"> for (ColumnFamilyDescriptor hcd : list) {</span>
+<span class="source-line-no">101</span><span id="line-101"> StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false);</span>
+<span class="source-line-no">102</span><span id="line-102"> String family = hcd.getNameAsString();</span>
+<span class="source-line-no">103</span><span id="line-103"> Path storePath = new Path(regionPath, family);</span>
+<span class="source-line-no">104</span><span id="line-104"> boolean succeed = false;</span>
+<span class="source-line-no">105</span><span id="line-105"> Set<String> regionMobs = new HashSet<String>();</span>
+<span class="source-line-no">106</span><span id="line-106"></span>
+<span class="source-line-no">107</span><span id="line-107"> while (!succeed) {</span>
+<span class="source-line-no">108</span><span id="line-108"> if (!fs.exists(storePath)) {</span>
+<span class="source-line-no">109</span><span id="line-109"> String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore"</span>
+<span class="source-line-no">110</span><span id="line-110"> + " execution, aborting MOB file cleaner chore.", storePath);</span>
+<span class="source-line-no">111</span><span id="line-111"> throw new IOException(errMsg);</span>
+<span class="source-line-no">112</span><span id="line-112"> }</span>
+<span class="source-line-no">113</span><span id="line-113"> List<StoreFileInfo> storeFileInfos = sft.load();</span>
+<span class="source-line-no">114</span><span id="line-114"> LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath);</span>
+<span class="source-line-no">115</span><span id="line-115"> Path currentPath = null;</span>
+<span class="source-line-no">116</span><span id="line-116"> try {</span>
+<span class="source-line-no">117</span><span id="line-117"> for (StoreFileInfo storeFileInfo : storeFileInfos) {</span>
+<span class="source-line-no">118</span><span id="line-118"> Path pp = storeFileInfo.getPath();</span>
+<span class="source-line-no">119</span><span id="line-119"> currentPath = pp;</span>
+<span class="source-line-no">120</span><span id="line-120"> LOG.trace("Store file: {}", pp);</span>
+<span class="source-line-no">121</span><span id="line-121"> HStoreFile sf = null;</span>
+<span class="source-line-no">122</span><span id="line-122"> byte[] mobRefData = null;</span>
+<span class="source-line-no">123</span><span id="line-123"> byte[] bulkloadMarkerData = null;</span>
+<span class="source-line-no">124</span><span id="line-124"> try {</span>
+<span class="source-line-no">125</span><span id="line-125"> sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED);</span>
+<span class="source-line-no">126</span><span id="line-126"> sf.initReader();</span>
+<span class="source-line-no">127</span><span id="line-127"> mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);</span>
+<span class="source-line-no">128</span><span id="line-128"> bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);</span>
+<span class="source-line-no">129</span><span id="line-129"> // close store file to avoid memory leaks</span>
+<span class="source-line-no">130</span><span id="line-130"> sf.closeStoreFile(true);</span>
+<span class="source-line-no">131</span><span id="line-131"> } catch (IOException ex) {</span>
+<span class="source-line-no">132</span><span id="line-132"> // When FileBased SFT is active the store dir can contain corrupted or incomplete</span>
+<span class="source-line-no">133</span><span id="line-133"> // files. So read errors are expected. We just skip these files.</span>
+<span class="source-line-no">134</span><span id="line-134"> if (ex instanceof FileNotFoundException) {</span>
+<span class="source-line-no">135</span><span id="line-135"> throw ex;</span>
+<span class="source-line-no">136</span><span id="line-136"> }</span>
+<span class="source-line-no">137</span><span id="line-137"> LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex);</span>
+<span class="source-line-no">138</span><span id="line-138"> continue;</span>
+<span class="source-line-no">139</span><span id="line-139"> }</span>
+<span class="source-line-no">140</span><span id="line-140"> if (mobRefData == null) {</span>
+<span class="source-line-no">141</span><span id="line-141"> if (bulkloadMarkerData == null) {</span>
+<span class="source-line-no">142</span><span id="line-142"> LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "</span>
+<span class="source-line-no">143</span><span id="line-143"> + "can not proceed until all old files will be MOB-compacted.", pp);</span>
+<span class="source-line-no">144</span><span id="line-144"> return;</span>
+<span class="source-line-no">145</span><span id="line-145"> } else {</span>
+<span class="source-line-no">146</span><span id="line-146"> LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);</span>
+<span class="source-line-no">147</span><span id="line-147"> continue;</span>
+<span class="source-line-no">148</span><span id="line-148"> }</span>
+<span class="source-line-no">149</span><span id="line-149"> }</span>
+<span class="source-line-no">150</span><span id="line-150"> // file may or may not have MOB references, but was created by the distributed</span>
+<span class="source-line-no">151</span><span id="line-151"> // mob compaction code.</span>
+<span class="source-line-no">152</span><span id="line-152"> try {</span>
+<span class="source-line-no">153</span><span id="line-153"> SetMultimap<TableName, String> mobs =</span>
+<span class="source-line-no">154</span><span id="line-154"> MobUtils.deserializeMobFileRefs(mobRefData).build();</span>
+<span class="source-line-no">155</span><span id="line-155"> LOG.debug("Found {} mob references for store={}", mobs.size(), sf);</span>
+<span class="source-line-no">156</span><span id="line-156"> LOG.trace("Specific mob references found for store={} : {}", sf, mobs);</span>
+<span class="source-line-no">157</span><span id="line-157"> regionMobs.addAll(mobs.values());</span>
+<span class="source-line-no">158</span><span id="line-158"> } catch (RuntimeException exception) {</span>
+<span class="source-line-no">159</span><span id="line-159"> throw new IOException("failure getting mob references for hfile " + sf, exception);</span>
+<span class="source-line-no">160</span><span id="line-160"> }</span>
+<span class="source-line-no">161</span><span id="line-161"> }</span>
+<span class="source-line-no">162</span><span id="line-162"> } catch (FileNotFoundException e) {</span>
+<span class="source-line-no">163</span><span id="line-163"> LOG.warn(</span>
+<span class="source-line-no">164</span><span id="line-164"> "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error",</span>
+<span class="source-line-no">165</span><span id="line-165"> currentPath, e);</span>
+<span class="source-line-no">166</span><span id="line-166"> regionMobs.clear();</span>
+<span class="source-line-no">167</span><span id="line-167"> continue;</span>
+<span class="source-line-no">168</span><span id="line-168"> }</span>
+<span class="source-line-no">169</span><span id="line-169"> succeed = true;</span>
+<span class="source-line-no">170</span><span id="line-170"> }</span>
+<span class="source-line-no">171</span><span id="line-171"></span>
+<span class="source-line-no">172</span><span id="line-172"> // Add MOB references for current region/family</span>
+<span class="source-line-no">173</span><span id="line-173"> allActiveMobFileName.addAll(regionMobs);</span>
+<span class="source-line-no">174</span><span id="line-174"> } // END column families</span>
+<span class="source-line-no">175</span><span id="line-175"> } // END regions</span>
+<span class="source-line-no">176</span><span id="line-176"> // Check if number of MOB files too big (over 1M)</span>
+<span class="source-line-no">177</span><span id="line-177"> if (allActiveMobFileName.size() > 1000000) {</span>
+<span class="source-line-no">178</span><span id="line-178"> LOG.warn("Found too many active MOB files: {}, table={}, "</span>
+<span class="source-line-no">179</span><span id="line-179"> + "this may result in high memory pressure.", allActiveMobFileName.size(), table);</span>
+<span class="source-line-no">180</span><span id="line-180"> }</span>
+<span class="source-line-no">181</span><span id="line-181"> LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);</span>
+<span class="source-line-no">182</span><span id="line-182"> allActiveMobFileName.stream().forEach(LOG::trace);</span>
+<span class="source-line-no">183</span><span id="line-183"></span>
+<span class="source-line-no">184</span><span id="line-184"> // Now scan MOB directories and find MOB files with no references to them</span>
+<span class="source-line-no">185</span><span id="line-185"> for (ColumnFamilyDescriptor hcd : list) {</span>
+<span class="source-line-no">186</span><span id="line-186"> checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames,</span>
+<span class="source-line-no">187</span><span id="line-187"> maxCreationTimeToArchive);</span>
+<span class="source-line-no">188</span><span id="line-188"> }</span>
+<span class="source-line-no">189</span><span id="line-189"> }</span>
+<span class="source-line-no">190</span><span id="line-190"></span>
+<span class="source-line-no">191</span><span id="line-191"> private static void checkColumnFamilyDescriptor(Configuration conf, TableName table,</span>
+<span class="source-line-no">192</span><span id="line-192"> FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set<String> regionNames,</span>
+<span class="source-line-no">193</span><span id="line-193"> long maxCreationTimeToArchive) throws IOException {</span>
+<span class="source-line-no">194</span><span id="line-194"> List<Path> toArchive = new ArrayList<Path>();</span>
+<span class="source-line-no">195</span><span id="line-195"> String family = hcd.getNameAsString();</span>
+<span class="source-line-no">196</span><span id="line-196"> Path dir = MobUtils.getMobFamilyPath(conf, table, family);</span>
+<span class="source-line-no">197</span><span id="line-197"> RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);</span>
+<span class="source-line-no">198</span><span id="line-198"> while (rit.hasNext()) {</span>
+<span class="source-line-no">199</span><span id="line-199"> LocatedFileStatus lfs = rit.next();</span>
+<span class="source-line-no">200</span><span id="line-200"> Path p = lfs.getPath();</span>
+<span class="source-line-no">201</span><span id="line-201"> String[] mobParts = p.getName().split("_");</span>
+<span class="source-line-no">202</span><span id="line-202"> String regionName = mobParts[mobParts.length - 1];</span>
+<span class="source-line-no">203</span><span id="line-203"></span>
+<span class="source-line-no">204</span><span id="line-204"> if (!regionNames.contains(regionName)) {</span>
+<span class="source-line-no">205</span><span id="line-205"> // MOB belonged to a region no longer hosted</span>
+<span class="source-line-no">206</span><span id="line-206"> long creationTime = fs.getFileStatus(p).getModificationTime();</span>
+<span class="source-line-no">207</span><span id="line-207"> if (creationTime < maxCreationTimeToArchive) {</span>
+<span class="source-line-no">208</span><span id="line-208"> LOG.trace("Archiving MOB file {} creation time={}", p,</span>
+<span class="source-line-no">209</span><span id="line-209"> (fs.getFileStatus(p).getModificationTime()));</span>
+<span class="source-line-no">210</span><span id="line-210"> toArchive.add(p);</span>
+<span class="source-line-no">211</span><span id="line-211"> } else {</span>
+<span class="source-line-no">212</span><span id="line-212"> LOG.trace("Skipping fresh file: {}. Creation time={}", p,</span>
+<span class="source-line-no">213</span><span id="line-213"> fs.getFileStatus(p).getModificationTime());</span>
+<span class="source-line-no">214</span><span id="line-214"> }</span>
+<span class="source-line-no">215</span><span id="line-215"> } else {</span>
+<span class="source-line-no">216</span><span id="line-216"> LOG.trace("Keeping MOB file with existing region: {}", p);</span>
+<span class="source-line-no">217</span><span id="line-217"> }</span>
+<span class="source-line-no">218</span><span id="line-218"> }</span>
+<span class="source-line-no">219</span><span id="line-219"> LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(),</span>
+<span class="source-line-no">220</span><span id="line-220"> table, family);</span>
+<span class="source-line-no">221</span><span id="line-221"> archiveMobFiles(conf, table, admin, family.getBytes(), toArchive);</span>
+<span class="source-line-no">222</span><span id="line-222"> LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);</span>
+<span class="source-line-no">223</span><span id="line-223"> }</span>
+<span class="source-line-no">224</span><span id="line-224"></span>
+<span class="source-line-no">225</span><span id="line-225"> /**</span>
+<span class="source-line-no">226</span><span id="line-226"> * Archives the mob files.</span>
+<span class="source-line-no">227</span><span id="line-227"> * @param conf The current configuration.</span>
+<span class="source-line-no">228</span><span id="line-228"> * @param tableName The table name.</span>
+<span class="source-line-no">229</span><span id="line-229"> * @param family The name of the column family.</span>
+<span class="source-line-no">230</span><span id="line-230"> * @param storeFiles The files to be archived.</span>
+<span class="source-line-no">231</span><span id="line-231"> * @throws IOException exception</span>
+<span class="source-line-no">232</span><span id="line-232"> */</span>
+<span class="source-line-no">233</span><span id="line-233"> private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin,</span>
+<span class="source-line-no">234</span><span id="line-234"> byte[] family, List<Path> storeFiles) throws IOException {</span>
+<span class="source-line-no">235</span><span id="line-235"></span>
+<span class="source-line-no">236</span><span id="line-236"> if (storeFiles.size() == 0) {</span>
+<span class="source-line-no">237</span><span id="line-237"> // nothing to remove</span>
+<span class="source-line-no">238</span><span id="line-238"> LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName,</span>
+<span class="source-line-no">239</span><span id="line-239"> Bytes.toString(family));</span>
+<span class="source-line-no">240</span><span id="line-240"> return;</span>
+<span class="source-line-no">241</span><span id="line-241"> }</span>
+<span class="source-line-no">242</span><span id="line-242"> Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);</span>
+<span class="source-line-no">243</span><span id="line-243"> FileSystem fs = storeFiles.get(0).getFileSystem(conf);</span>
+<span class="source-line-no">244</span><span id="line-244"></span>
+<span class="source-line-no">245</span><span id="line-245"> for (Path p : storeFiles) {</span>
+<span class="source-line-no">246</span><span id="line-246"> LOG.debug("MOB Cleaner is archiving: {}", p);</span>
+<span class="source-line-no">247</span><span id="line-247"> HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,</span>
+<span class="source-line-no">248</span><span id="line-248"> family, p);</span>
+<span class="source-line-no">249</span><span id="line-249"> }</span>
+<span class="source-line-no">250</span><span id="line-250"> }</span>
+<span class="source-line-no">251</span><span id="line-251">}</span>
diff --git a/apidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html b/apidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
index d0c93fa6d3c..b660f6a4e16 100644
--- a/apidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
+++ b/apidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
@@ -604,24 +604,32 @@
<span class="source-line-no">591</span><span id="line-591"> + (3 * Bytes.SIZEOF_INT) + (4 * Bytes.SIZEOF_LONG));</span>
<span class="source-line-no">592</span><span id="line-592"></span>
<span class="source-line-no">593</span><span id="line-593"> /**</span>
-<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL.</span>
-<span class="source-line-no">595</span><span id="line-595"> */</span>
-<span class="source-line-no">596</span><span id="line-596"> @Override</span>
-<span class="source-line-no">597</span><span id="line-597"> DatanodeInfo[] getPipeline() {</span>
-<span class="source-line-no">598</span><span id="line-598"> if (this.hdfs_out != null) {</span>
-<span class="source-line-no">599</span><span id="line-599"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
-<span class="source-line-no">600</span><span id="line-600"> return ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
-<span class="source-line-no">601</span><span id="line-601"> }</span>
-<span class="source-line-no">602</span><span id="line-602"> }</span>
-<span class="source-line-no">603</span><span id="line-603"> return new DatanodeInfo[0];</span>
-<span class="source-line-no">604</span><span id="line-604"> }</span>
-<span class="source-line-no">605</span><span id="line-605"></span>
-<span class="source-line-no">606</span><span id="line-606"> @Override</span>
-<span class="source-line-no">607</span><span id="line-607"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
-<span class="source-line-no">608</span><span id="line-608"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
-<span class="source-line-no">609</span><span id="line-609"> return CombinedWriter.create(remoteWriter, localWriter);</span>
-<span class="source-line-no">610</span><span id="line-610"> }</span>
-<span class="source-line-no">611</span><span id="line-611">}</span>
+<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL. Note that DFSOutputStream#getPipeline() can</span>
+<span class="source-line-no">595</span><span id="line-595"> * legitimately return null (e.g. the underlying streamer is closed, or no block pipeline is</span>
+<span class="source-line-no">596</span><span id="line-596"> * currently established, such as between blocks) -- see HDFS-826. We normalize that to an empty</span>
+<span class="source-line-no">597</span><span id="line-597"> * array here so that callers (in particular debug-log formatting in AbstractFSWAL) never have to</span>
+<span class="source-line-no">598</span><span id="line-598"> * null-check, matching the contract already honored by AsyncFSWAL#getPipeline().</span>
+<span class="source-line-no">599</span><span id="line-599"> */</span>
+<span class="source-line-no">600</span><span id="line-600"> @Override</span>
+<span class="source-line-no">601</span><span id="line-601"> DatanodeInfo[] getPipeline() {</span>
+<span class="source-line-no">602</span><span id="line-602"> if (this.hdfs_out != null) {</span>
+<span class="source-line-no">603</span><span id="line-603"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
+<span class="source-line-no">604</span><span id="line-604"> DatanodeInfo[] pipeline =</span>
+<span class="source-line-no">605</span><span id="line-605"> ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
+<span class="source-line-no">606</span><span id="line-606"> if (pipeline != null) {</span>
+<span class="source-line-no">607</span><span id="line-607"> return pipeline;</span>
+<span class="source-line-no">608</span><span id="line-608"> }</span>
+<span class="source-line-no">609</span><span id="line-609"> }</span>
+<span class="source-line-no">610</span><span id="line-610"> }</span>
+<span class="source-line-no">611</span><span id="line-611"> return new DatanodeInfo[0];</span>
+<span class="source-line-no">612</span><span id="line-612"> }</span>
+<span class="source-line-no">613</span><span id="line-613"></span>
+<span class="source-line-no">614</span><span id="line-614"> @Override</span>
+<span class="source-line-no">615</span><span id="line-615"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
+<span class="source-line-no">616</span><span id="line-616"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
+<span class="source-line-no">617</span><span id="line-617"> return CombinedWriter.create(remoteWriter, localWriter);</span>
+<span class="source-line-no">618</span><span id="line-618"> }</span>
+<span class="source-line-no">619</span><span id="line-619">}</span>
diff --git a/books/apache-hbase-reference-guide-dark-mode.pdf b/books/apache-hbase-reference-guide-dark-mode.pdf
index 715a467ce66..8e3d934fb09 100644
Binary files a/books/apache-hbase-reference-guide-dark-mode.pdf and b/books/apache-hbase-reference-guide-dark-mode.pdf differ
diff --git a/books/apache-hbase-reference-guide.pdf b/books/apache-hbase-reference-guide.pdf
index bc7f2165ffc..5e099cc4f4e 100644
Binary files a/books/apache-hbase-reference-guide.pdf and b/books/apache-hbase-reference-guide.pdf differ
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 436954da18f..c895333bcd7 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-checkstyle-plugin:3.6.0:checkstyle-aggregate at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-checkstyle-plugin:3.6.0:checkstyle-aggregate at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/dependency-convergence.html b/dependency-convergence.html
index ccfa3d8daa3..4ba33912a55 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-convergence at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-convergence at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/dependency-info.html b/dependency-info.html
index 43b34b5ffec..4f4461436ef 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-info at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-info at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/dependency-management.html b/dependency-management.html
index e094fff2199..314bf3029cb 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/devapidocs/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html b/devapidocs/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
index 5690e003f14..1d19cf85bbb 100644
--- a/devapidocs/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
+++ b/devapidocs/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
@@ -78,7 +78,7 @@ loadScripts(document, 'script');</script>
<section class="class-description" id="class-description">
<hr>
<div class="type-signature"><span class="annotations">@Private
-</span><span class="modifiers">public final class </span><span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-53">MobFileCleanupUtil</a></span>
+</span><span class="modifiers">public final class </span><span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-54">MobFileCleanupUtil</a></span>
<span class="extends-implements">extends <a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Object.html" title="class or interface in java.lang" class="external-link">Object</a></span></div>
</section>
<section class="summary">
@@ -169,7 +169,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="LOG">
<h3>LOG</h3>
-<div class="member-signature"><span class="modifiers">private static final</span> <span class="return-type">org.slf4j.Logger</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-56">LOG</a></span></div>
+<div class="member-signature"><span class="modifiers">private static final</span> <span class="return-type">org.slf4j.Logger</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-57">LOG</a></span></div>
</section>
</li>
</ul>
@@ -183,7 +183,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="<init>()">
<h3>MobFileCleanupUtil</h3>
-<div class="member-signature"><span class="modifiers">private</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-58">MobFileCleanupUtil</a></span>()</div>
+<div class="member-signature"><span class="modifiers">private</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-59">MobFileCleanupUtil</a></span>()</div>
</section>
</li>
</ul>
@@ -197,7 +197,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="cleanupObsoleteMobFiles(org.apache.hadoop.conf.Configuration,org.apache.hadoop.hbase.TableName,org.apache.hadoop.hbase.client.Admin)">
<h3>cleanupObsoleteMobFiles</h3>
-<div class="member-signature"><span class="modifiers">public static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-67">cleanupObsoleteMobFiles</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
+<div class="member-signature"><span class="modifiers">public static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-68">cleanupObsoleteMobFiles</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
<a href="../TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> table,
<a href="../client/Admin.html" title="interface in org.apache.hadoop.hbase.client">Admin</a> admin)</span>
throws <span class="exceptions"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/IOException.html" title="class or interface in java.io" class="external-link">IOException</a></span></div>
@@ -214,7 +214,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="checkColumnFamilyDescriptor(org.apache.hadoop.conf.Configuration,org.apache.hadoop.hbase.TableName,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.Admin,org.apache.hadoop.hbase.client.ColumnFamilyDescriptor,java.util.Set,long)">
<h3>checkColumnFamilyDescriptor</h3>
-<div class="member-signature"><span class="modifiers">private static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-190">checkColumnFamilyDescriptor</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
+<div class="member-signature"><span class="modifiers">private static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-191">checkColumnFamilyDescriptor</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
<a href="../TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> table,
org.apache.hadoop.fs.FileSystem fs,
<a href="../client/Admin.html" title="interface in org.apache.hadoop.hbase.client">Admin</a> admin,
@@ -231,7 +231,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="archiveMobFiles(org.apache.hadoop.conf.Configuration,org.apache.hadoop.hbase.TableName,org.apache.hadoop.hbase.client.Admin,byte[],java.util.List)">
<h3>archiveMobFiles</h3>
-<div class="member-signature"><span class="modifiers">private static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-232">archiveMobFiles</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
+<div class="member-signature"><span class="modifiers">private static</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html#line-233">archiveMobFiles</a></span><wbr><span class="parameters">(org.apache.hadoop.conf.Configuration conf,
<a href="../TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> tableName,
<a href="../client/Admin.html" title="interface in org.apache.hadoop.hbase.client">Admin</a> admin,
byte[] family,
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
index 7570c18159c..c6f839d04e1 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
@@ -840,8 +840,12 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="getPipeline()">
<h3>getPipeline</h3>
-<div class="member-signature"><span class="return-type">org.apache.hadoop.hdfs.protocol.DatanodeInfo[]</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html#line-596">getPipeline</a></span>()</div>
-<div class="block">This method gets the pipeline for the current WAL.</div>
+<div class="member-signature"><span class="return-type">org.apache.hadoop.hdfs.protocol.DatanodeInfo[]</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html#line-600">getPipeline</a></span>()</div>
+<div class="block">This method gets the pipeline for the current WAL. Note that DFSOutputStream#getPipeline() can
+ legitimately return null (e.g. the underlying streamer is closed, or no block pipeline is
+ currently established, such as between blocks) -- see HDFS-826. We normalize that to an empty
+ array here so that callers (in particular debug-log formatting in AbstractFSWAL) never have to
+ null-check, matching the contract already honored by AsyncFSWAL#getPipeline().</div>
<dl class="notes">
<dt>Specified by:</dt>
<dd><code><a href="AbstractFSWAL.html#getPipeline()">getPipeline</a></code> in class <code><a href="AbstractFSWAL.html" title="class in org.apache.hadoop.hbase.regionserver.wal">AbstractFSWAL</a><<a href="../../wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a>></code></dd>
@@ -851,7 +855,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="createCombinedWriter(org.apache.hadoop.hbase.wal.WALProvider.Writer,org.apache.hadoop.hbase.wal.WALProvider.Writer)">
<h3>createCombinedWriter</h3>
-<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type"><a href="../../wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html#line-606">createCombinedWriter</a></span><wbr><span class="parameters">(<a href="../../wal/WALProvider.Writer.html" title="interface in org.apache [...]
+<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type"><a href="../../wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html#line-614">createCombinedWriter</a></span><wbr><span class="parameters">(<a href="../../wal/WALProvider.Writer.html" title="interface in org.apache [...]
<a href="../../wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> remoteWriter)</span></div>
<dl class="notes">
<dt>Specified by:</dt>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html b/devapidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
index f247284b008..d9c748162fa 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.html
@@ -33,234 +33,235 @@
<span class="source-line-no">020</span><span id="line-20">import java.io.FileNotFoundException;</span>
<span class="source-line-no">021</span><span id="line-21">import java.io.IOException;</span>
<span class="source-line-no">022</span><span id="line-22">import java.util.ArrayList;</span>
-<span class="source-line-no">023</span><span id="line-23">import java.util.HashSet;</span>
-<span class="source-line-no">024</span><span id="line-24">import java.util.List;</span>
-<span class="source-line-no">025</span><span id="line-25">import java.util.Set;</span>
-<span class="source-line-no">026</span><span id="line-26">import org.apache.hadoop.conf.Configuration;</span>
-<span class="source-line-no">027</span><span id="line-27">import org.apache.hadoop.fs.FileSystem;</span>
-<span class="source-line-no">028</span><span id="line-28">import org.apache.hadoop.fs.LocatedFileStatus;</span>
-<span class="source-line-no">029</span><span id="line-29">import org.apache.hadoop.fs.Path;</span>
-<span class="source-line-no">030</span><span id="line-30">import org.apache.hadoop.fs.RemoteIterator;</span>
-<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.hbase.TableName;</span>
-<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.hbase.backup.HFileArchiver;</span>
-<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.hbase.client.Admin;</span>
-<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span>
-<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
-<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span>
-<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.regionserver.BloomType;</span>
-<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;</span>
-<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.regionserver.HStoreFile;</span>
-<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span>
-<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;</span>
-<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;</span>
-<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.util.Bytes;</span>
-<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
-<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span>
-<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.util.FSUtils;</span>
-<span class="source-line-no">047</span><span id="line-47">import org.apache.yetus.audience.InterfaceAudience;</span>
-<span class="source-line-no">048</span><span id="line-48">import org.slf4j.Logger;</span>
-<span class="source-line-no">049</span><span id="line-49">import org.slf4j.LoggerFactory;</span>
-<span class="source-line-no">050</span><span id="line-50"></span>
-<span class="source-line-no">051</span><span id="line-51">import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap;</span>
-<span class="source-line-no">052</span><span id="line-52"></span>
-<span class="source-line-no">053</span><span id="line-53">@InterfaceAudience.Private</span>
-<span class="source-line-no">054</span><span id="line-54">public final class MobFileCleanupUtil {</span>
-<span class="source-line-no">055</span><span id="line-55"></span>
-<span class="source-line-no">056</span><span id="line-56"> private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class);</span>
-<span class="source-line-no">057</span><span id="line-57"></span>
-<span class="source-line-no">058</span><span id="line-58"> private MobFileCleanupUtil() {</span>
-<span class="source-line-no">059</span><span id="line-59"> }</span>
-<span class="source-line-no">060</span><span id="line-60"></span>
-<span class="source-line-no">061</span><span id="line-61"> /**</span>
-<span class="source-line-no">062</span><span id="line-62"> * Performs housekeeping file cleaning (called by MOB Cleaner chore)</span>
-<span class="source-line-no">063</span><span id="line-63"> * @param conf configuration</span>
-<span class="source-line-no">064</span><span id="line-64"> * @param table table name</span>
-<span class="source-line-no">065</span><span id="line-65"> * @throws IOException exception</span>
-<span class="source-line-no">066</span><span id="line-66"> */</span>
-<span class="source-line-no">067</span><span id="line-67"> public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin)</span>
-<span class="source-line-no">068</span><span id="line-68"> throws IOException {</span>
-<span class="source-line-no">069</span><span id="line-69"> long minAgeToArchive =</span>
-<span class="source-line-no">070</span><span id="line-70"> conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);</span>
-<span class="source-line-no">071</span><span id="line-71"> // We check only those MOB files, which creation time is less</span>
-<span class="source-line-no">072</span><span id="line-72"> // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap</span>
-<span class="source-line-no">073</span><span id="line-73"> // gives us full confidence that all corresponding store files will</span>
-<span class="source-line-no">074</span><span id="line-74"> // exist at the time cleaning procedure begins and will be examined.</span>
-<span class="source-line-no">075</span><span id="line-75"> // So, if MOB file creation time is greater than this maxTimeToArchive,</span>
-<span class="source-line-no">076</span><span id="line-76"> // this will be skipped and won't be archived.</span>
-<span class="source-line-no">077</span><span id="line-77"> long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;</span>
-<span class="source-line-no">078</span><span id="line-78"> TableDescriptor htd = admin.getDescriptor(table);</span>
-<span class="source-line-no">079</span><span id="line-79"> List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);</span>
-<span class="source-line-no">080</span><span id="line-80"> if (list.size() == 0) {</span>
-<span class="source-line-no">081</span><span id="line-81"> LOG.info("Skipping non-MOB table [{}]", table);</span>
-<span class="source-line-no">082</span><span id="line-82"> return;</span>
-<span class="source-line-no">083</span><span id="line-83"> } else {</span>
-<span class="source-line-no">084</span><span id="line-84"> LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",</span>
-<span class="source-line-no">085</span><span id="line-85"> maxCreationTimeToArchive, table);</span>
-<span class="source-line-no">086</span><span id="line-86"> }</span>
-<span class="source-line-no">087</span><span id="line-87"></span>
-<span class="source-line-no">088</span><span id="line-88"> FileSystem fs = FileSystem.get(conf);</span>
-<span class="source-line-no">089</span><span id="line-89"> Set<String> regionNames = new HashSet<>();</span>
-<span class="source-line-no">090</span><span id="line-90"> Path rootDir = CommonFSUtils.getRootDir(conf);</span>
-<span class="source-line-no">091</span><span id="line-91"> Path tableDir = CommonFSUtils.getTableDir(rootDir, table);</span>
-<span class="source-line-no">092</span><span id="line-92"> List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);</span>
-<span class="source-line-no">093</span><span id="line-93"></span>
-<span class="source-line-no">094</span><span id="line-94"> Set<String> allActiveMobFileName = new HashSet<String>();</span>
-<span class="source-line-no">095</span><span id="line-95"> for (Path regionPath : regionDirs) {</span>
-<span class="source-line-no">096</span><span id="line-96"> regionNames.add(regionPath.getName());</span>
-<span class="source-line-no">097</span><span id="line-97"> HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir,</span>
-<span class="source-line-no">098</span><span id="line-98"> HRegionFileSystem.loadRegionInfoFileContent(fs, regionPath));</span>
-<span class="source-line-no">099</span><span id="line-99"> for (ColumnFamilyDescriptor hcd : list) {</span>
-<span class="source-line-no">100</span><span id="line-100"> StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false);</span>
-<span class="source-line-no">101</span><span id="line-101"> String family = hcd.getNameAsString();</span>
-<span class="source-line-no">102</span><span id="line-102"> Path storePath = new Path(regionPath, family);</span>
-<span class="source-line-no">103</span><span id="line-103"> boolean succeed = false;</span>
-<span class="source-line-no">104</span><span id="line-104"> Set<String> regionMobs = new HashSet<String>();</span>
-<span class="source-line-no">105</span><span id="line-105"></span>
-<span class="source-line-no">106</span><span id="line-106"> while (!succeed) {</span>
-<span class="source-line-no">107</span><span id="line-107"> if (!fs.exists(storePath)) {</span>
-<span class="source-line-no">108</span><span id="line-108"> String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore"</span>
-<span class="source-line-no">109</span><span id="line-109"> + " execution, aborting MOB file cleaner chore.", storePath);</span>
-<span class="source-line-no">110</span><span id="line-110"> throw new IOException(errMsg);</span>
-<span class="source-line-no">111</span><span id="line-111"> }</span>
-<span class="source-line-no">112</span><span id="line-112"> List<StoreFileInfo> storeFileInfos = sft.load();</span>
-<span class="source-line-no">113</span><span id="line-113"> LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath);</span>
-<span class="source-line-no">114</span><span id="line-114"> Path currentPath = null;</span>
-<span class="source-line-no">115</span><span id="line-115"> try {</span>
-<span class="source-line-no">116</span><span id="line-116"> for (StoreFileInfo storeFileInfo : storeFileInfos) {</span>
-<span class="source-line-no">117</span><span id="line-117"> Path pp = storeFileInfo.getPath();</span>
-<span class="source-line-no">118</span><span id="line-118"> currentPath = pp;</span>
-<span class="source-line-no">119</span><span id="line-119"> LOG.trace("Store file: {}", pp);</span>
-<span class="source-line-no">120</span><span id="line-120"> HStoreFile sf = null;</span>
-<span class="source-line-no">121</span><span id="line-121"> byte[] mobRefData = null;</span>
-<span class="source-line-no">122</span><span id="line-122"> byte[] bulkloadMarkerData = null;</span>
-<span class="source-line-no">123</span><span id="line-123"> try {</span>
-<span class="source-line-no">124</span><span id="line-124"> sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED);</span>
-<span class="source-line-no">125</span><span id="line-125"> sf.initReader();</span>
-<span class="source-line-no">126</span><span id="line-126"> mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);</span>
-<span class="source-line-no">127</span><span id="line-127"> bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);</span>
-<span class="source-line-no">128</span><span id="line-128"> // close store file to avoid memory leaks</span>
-<span class="source-line-no">129</span><span id="line-129"> sf.closeStoreFile(true);</span>
-<span class="source-line-no">130</span><span id="line-130"> } catch (IOException ex) {</span>
-<span class="source-line-no">131</span><span id="line-131"> // When FileBased SFT is active the store dir can contain corrupted or incomplete</span>
-<span class="source-line-no">132</span><span id="line-132"> // files. So read errors are expected. We just skip these files.</span>
-<span class="source-line-no">133</span><span id="line-133"> if (ex instanceof FileNotFoundException) {</span>
-<span class="source-line-no">134</span><span id="line-134"> throw ex;</span>
-<span class="source-line-no">135</span><span id="line-135"> }</span>
-<span class="source-line-no">136</span><span id="line-136"> LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex);</span>
-<span class="source-line-no">137</span><span id="line-137"> continue;</span>
-<span class="source-line-no">138</span><span id="line-138"> }</span>
-<span class="source-line-no">139</span><span id="line-139"> if (mobRefData == null) {</span>
-<span class="source-line-no">140</span><span id="line-140"> if (bulkloadMarkerData == null) {</span>
-<span class="source-line-no">141</span><span id="line-141"> LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "</span>
-<span class="source-line-no">142</span><span id="line-142"> + "can not proceed until all old files will be MOB-compacted.", pp);</span>
-<span class="source-line-no">143</span><span id="line-143"> return;</span>
-<span class="source-line-no">144</span><span id="line-144"> } else {</span>
-<span class="source-line-no">145</span><span id="line-145"> LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);</span>
-<span class="source-line-no">146</span><span id="line-146"> continue;</span>
-<span class="source-line-no">147</span><span id="line-147"> }</span>
-<span class="source-line-no">148</span><span id="line-148"> }</span>
-<span class="source-line-no">149</span><span id="line-149"> // file may or may not have MOB references, but was created by the distributed</span>
-<span class="source-line-no">150</span><span id="line-150"> // mob compaction code.</span>
-<span class="source-line-no">151</span><span id="line-151"> try {</span>
-<span class="source-line-no">152</span><span id="line-152"> SetMultimap<TableName, String> mobs =</span>
-<span class="source-line-no">153</span><span id="line-153"> MobUtils.deserializeMobFileRefs(mobRefData).build();</span>
-<span class="source-line-no">154</span><span id="line-154"> LOG.debug("Found {} mob references for store={}", mobs.size(), sf);</span>
-<span class="source-line-no">155</span><span id="line-155"> LOG.trace("Specific mob references found for store={} : {}", sf, mobs);</span>
-<span class="source-line-no">156</span><span id="line-156"> regionMobs.addAll(mobs.values());</span>
-<span class="source-line-no">157</span><span id="line-157"> } catch (RuntimeException exception) {</span>
-<span class="source-line-no">158</span><span id="line-158"> throw new IOException("failure getting mob references for hfile " + sf, exception);</span>
-<span class="source-line-no">159</span><span id="line-159"> }</span>
-<span class="source-line-no">160</span><span id="line-160"> }</span>
-<span class="source-line-no">161</span><span id="line-161"> } catch (FileNotFoundException e) {</span>
-<span class="source-line-no">162</span><span id="line-162"> LOG.warn(</span>
-<span class="source-line-no">163</span><span id="line-163"> "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error",</span>
-<span class="source-line-no">164</span><span id="line-164"> currentPath, e);</span>
-<span class="source-line-no">165</span><span id="line-165"> regionMobs.clear();</span>
-<span class="source-line-no">166</span><span id="line-166"> continue;</span>
-<span class="source-line-no">167</span><span id="line-167"> }</span>
-<span class="source-line-no">168</span><span id="line-168"> succeed = true;</span>
-<span class="source-line-no">169</span><span id="line-169"> }</span>
-<span class="source-line-no">170</span><span id="line-170"></span>
-<span class="source-line-no">171</span><span id="line-171"> // Add MOB references for current region/family</span>
-<span class="source-line-no">172</span><span id="line-172"> allActiveMobFileName.addAll(regionMobs);</span>
-<span class="source-line-no">173</span><span id="line-173"> } // END column families</span>
-<span class="source-line-no">174</span><span id="line-174"> } // END regions</span>
-<span class="source-line-no">175</span><span id="line-175"> // Check if number of MOB files too big (over 1M)</span>
-<span class="source-line-no">176</span><span id="line-176"> if (allActiveMobFileName.size() > 1000000) {</span>
-<span class="source-line-no">177</span><span id="line-177"> LOG.warn("Found too many active MOB files: {}, table={}, "</span>
-<span class="source-line-no">178</span><span id="line-178"> + "this may result in high memory pressure.", allActiveMobFileName.size(), table);</span>
-<span class="source-line-no">179</span><span id="line-179"> }</span>
-<span class="source-line-no">180</span><span id="line-180"> LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);</span>
-<span class="source-line-no">181</span><span id="line-181"> allActiveMobFileName.stream().forEach(LOG::trace);</span>
-<span class="source-line-no">182</span><span id="line-182"></span>
-<span class="source-line-no">183</span><span id="line-183"> // Now scan MOB directories and find MOB files with no references to them</span>
-<span class="source-line-no">184</span><span id="line-184"> for (ColumnFamilyDescriptor hcd : list) {</span>
-<span class="source-line-no">185</span><span id="line-185"> checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames,</span>
-<span class="source-line-no">186</span><span id="line-186"> maxCreationTimeToArchive);</span>
-<span class="source-line-no">187</span><span id="line-187"> }</span>
-<span class="source-line-no">188</span><span id="line-188"> }</span>
-<span class="source-line-no">189</span><span id="line-189"></span>
-<span class="source-line-no">190</span><span id="line-190"> private static void checkColumnFamilyDescriptor(Configuration conf, TableName table,</span>
-<span class="source-line-no">191</span><span id="line-191"> FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set<String> regionNames,</span>
-<span class="source-line-no">192</span><span id="line-192"> long maxCreationTimeToArchive) throws IOException {</span>
-<span class="source-line-no">193</span><span id="line-193"> List<Path> toArchive = new ArrayList<Path>();</span>
-<span class="source-line-no">194</span><span id="line-194"> String family = hcd.getNameAsString();</span>
-<span class="source-line-no">195</span><span id="line-195"> Path dir = MobUtils.getMobFamilyPath(conf, table, family);</span>
-<span class="source-line-no">196</span><span id="line-196"> RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);</span>
-<span class="source-line-no">197</span><span id="line-197"> while (rit.hasNext()) {</span>
-<span class="source-line-no">198</span><span id="line-198"> LocatedFileStatus lfs = rit.next();</span>
-<span class="source-line-no">199</span><span id="line-199"> Path p = lfs.getPath();</span>
-<span class="source-line-no">200</span><span id="line-200"> String[] mobParts = p.getName().split("_");</span>
-<span class="source-line-no">201</span><span id="line-201"> String regionName = mobParts[mobParts.length - 1];</span>
-<span class="source-line-no">202</span><span id="line-202"></span>
-<span class="source-line-no">203</span><span id="line-203"> if (!regionNames.contains(regionName)) {</span>
-<span class="source-line-no">204</span><span id="line-204"> // MOB belonged to a region no longer hosted</span>
-<span class="source-line-no">205</span><span id="line-205"> long creationTime = fs.getFileStatus(p).getModificationTime();</span>
-<span class="source-line-no">206</span><span id="line-206"> if (creationTime < maxCreationTimeToArchive) {</span>
-<span class="source-line-no">207</span><span id="line-207"> LOG.trace("Archiving MOB file {} creation time={}", p,</span>
-<span class="source-line-no">208</span><span id="line-208"> (fs.getFileStatus(p).getModificationTime()));</span>
-<span class="source-line-no">209</span><span id="line-209"> toArchive.add(p);</span>
-<span class="source-line-no">210</span><span id="line-210"> } else {</span>
-<span class="source-line-no">211</span><span id="line-211"> LOG.trace("Skipping fresh file: {}. Creation time={}", p,</span>
-<span class="source-line-no">212</span><span id="line-212"> fs.getFileStatus(p).getModificationTime());</span>
-<span class="source-line-no">213</span><span id="line-213"> }</span>
-<span class="source-line-no">214</span><span id="line-214"> } else {</span>
-<span class="source-line-no">215</span><span id="line-215"> LOG.trace("Keeping MOB file with existing region: {}", p);</span>
-<span class="source-line-no">216</span><span id="line-216"> }</span>
-<span class="source-line-no">217</span><span id="line-217"> }</span>
-<span class="source-line-no">218</span><span id="line-218"> LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(),</span>
-<span class="source-line-no">219</span><span id="line-219"> table, family);</span>
-<span class="source-line-no">220</span><span id="line-220"> archiveMobFiles(conf, table, admin, family.getBytes(), toArchive);</span>
-<span class="source-line-no">221</span><span id="line-221"> LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);</span>
-<span class="source-line-no">222</span><span id="line-222"> }</span>
-<span class="source-line-no">223</span><span id="line-223"></span>
-<span class="source-line-no">224</span><span id="line-224"> /**</span>
-<span class="source-line-no">225</span><span id="line-225"> * Archives the mob files.</span>
-<span class="source-line-no">226</span><span id="line-226"> * @param conf The current configuration.</span>
-<span class="source-line-no">227</span><span id="line-227"> * @param tableName The table name.</span>
-<span class="source-line-no">228</span><span id="line-228"> * @param family The name of the column family.</span>
-<span class="source-line-no">229</span><span id="line-229"> * @param storeFiles The files to be archived.</span>
-<span class="source-line-no">230</span><span id="line-230"> * @throws IOException exception</span>
-<span class="source-line-no">231</span><span id="line-231"> */</span>
-<span class="source-line-no">232</span><span id="line-232"> private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin,</span>
-<span class="source-line-no">233</span><span id="line-233"> byte[] family, List<Path> storeFiles) throws IOException {</span>
-<span class="source-line-no">234</span><span id="line-234"></span>
-<span class="source-line-no">235</span><span id="line-235"> if (storeFiles.size() == 0) {</span>
-<span class="source-line-no">236</span><span id="line-236"> // nothing to remove</span>
-<span class="source-line-no">237</span><span id="line-237"> LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName,</span>
-<span class="source-line-no">238</span><span id="line-238"> Bytes.toString(family));</span>
-<span class="source-line-no">239</span><span id="line-239"> return;</span>
-<span class="source-line-no">240</span><span id="line-240"> }</span>
-<span class="source-line-no">241</span><span id="line-241"> Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);</span>
-<span class="source-line-no">242</span><span id="line-242"> FileSystem fs = storeFiles.get(0).getFileSystem(conf);</span>
-<span class="source-line-no">243</span><span id="line-243"></span>
-<span class="source-line-no">244</span><span id="line-244"> for (Path p : storeFiles) {</span>
-<span class="source-line-no">245</span><span id="line-245"> LOG.debug("MOB Cleaner is archiving: {}", p);</span>
-<span class="source-line-no">246</span><span id="line-246"> HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,</span>
-<span class="source-line-no">247</span><span id="line-247"> family, p);</span>
-<span class="source-line-no">248</span><span id="line-248"> }</span>
-<span class="source-line-no">249</span><span id="line-249"> }</span>
-<span class="source-line-no">250</span><span id="line-250">}</span>
+<span class="source-line-no">023</span><span id="line-23">import java.util.Date;</span>
+<span class="source-line-no">024</span><span id="line-24">import java.util.HashSet;</span>
+<span class="source-line-no">025</span><span id="line-25">import java.util.List;</span>
+<span class="source-line-no">026</span><span id="line-26">import java.util.Set;</span>
+<span class="source-line-no">027</span><span id="line-27">import org.apache.hadoop.conf.Configuration;</span>
+<span class="source-line-no">028</span><span id="line-28">import org.apache.hadoop.fs.FileSystem;</span>
+<span class="source-line-no">029</span><span id="line-29">import org.apache.hadoop.fs.LocatedFileStatus;</span>
+<span class="source-line-no">030</span><span id="line-30">import org.apache.hadoop.fs.Path;</span>
+<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.fs.RemoteIterator;</span>
+<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.hbase.TableName;</span>
+<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.hbase.backup.HFileArchiver;</span>
+<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.client.Admin;</span>
+<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;</span>
+<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
+<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.io.hfile.CacheConfig;</span>
+<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.regionserver.BloomType;</span>
+<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;</span>
+<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.regionserver.HStoreFile;</span>
+<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.regionserver.StoreFileInfo;</span>
+<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;</span>
+<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;</span>
+<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.util.Bytes;</span>
+<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
+<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;</span>
+<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.util.FSUtils;</span>
+<span class="source-line-no">048</span><span id="line-48">import org.apache.yetus.audience.InterfaceAudience;</span>
+<span class="source-line-no">049</span><span id="line-49">import org.slf4j.Logger;</span>
+<span class="source-line-no">050</span><span id="line-50">import org.slf4j.LoggerFactory;</span>
+<span class="source-line-no">051</span><span id="line-51"></span>
+<span class="source-line-no">052</span><span id="line-52">import org.apache.hbase.thirdparty.com.google.common.collect.SetMultimap;</span>
+<span class="source-line-no">053</span><span id="line-53"></span>
+<span class="source-line-no">054</span><span id="line-54">@InterfaceAudience.Private</span>
+<span class="source-line-no">055</span><span id="line-55">public final class MobFileCleanupUtil {</span>
+<span class="source-line-no">056</span><span id="line-56"></span>
+<span class="source-line-no">057</span><span id="line-57"> private static final Logger LOG = LoggerFactory.getLogger(MobFileCleanupUtil.class);</span>
+<span class="source-line-no">058</span><span id="line-58"></span>
+<span class="source-line-no">059</span><span id="line-59"> private MobFileCleanupUtil() {</span>
+<span class="source-line-no">060</span><span id="line-60"> }</span>
+<span class="source-line-no">061</span><span id="line-61"></span>
+<span class="source-line-no">062</span><span id="line-62"> /**</span>
+<span class="source-line-no">063</span><span id="line-63"> * Performs housekeeping file cleaning (called by MOB Cleaner chore)</span>
+<span class="source-line-no">064</span><span id="line-64"> * @param conf configuration</span>
+<span class="source-line-no">065</span><span id="line-65"> * @param table table name</span>
+<span class="source-line-no">066</span><span id="line-66"> * @throws IOException exception</span>
+<span class="source-line-no">067</span><span id="line-67"> */</span>
+<span class="source-line-no">068</span><span id="line-68"> public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Admin admin)</span>
+<span class="source-line-no">069</span><span id="line-69"> throws IOException {</span>
+<span class="source-line-no">070</span><span id="line-70"> long minAgeToArchive =</span>
+<span class="source-line-no">071</span><span id="line-71"> conf.getLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, MobConstants.DEFAULT_MIN_AGE_TO_ARCHIVE);</span>
+<span class="source-line-no">072</span><span id="line-72"> // We check only those MOB files, which creation time is less</span>
+<span class="source-line-no">073</span><span id="line-73"> // than maxCreationTimeToArchive. This is a current time - 1h. 1 hour gap</span>
+<span class="source-line-no">074</span><span id="line-74"> // gives us full confidence that all corresponding store files will</span>
+<span class="source-line-no">075</span><span id="line-75"> // exist at the time cleaning procedure begins and will be examined.</span>
+<span class="source-line-no">076</span><span id="line-76"> // So, if MOB file creation time is greater than this maxTimeToArchive,</span>
+<span class="source-line-no">077</span><span id="line-77"> // this will be skipped and won't be archived.</span>
+<span class="source-line-no">078</span><span id="line-78"> long maxCreationTimeToArchive = EnvironmentEdgeManager.currentTime() - minAgeToArchive;</span>
+<span class="source-line-no">079</span><span id="line-79"> TableDescriptor htd = admin.getDescriptor(table);</span>
+<span class="source-line-no">080</span><span id="line-80"> List<ColumnFamilyDescriptor> list = MobUtils.getMobColumnFamilies(htd);</span>
+<span class="source-line-no">081</span><span id="line-81"> if (list.size() == 0) {</span>
+<span class="source-line-no">082</span><span id="line-82"> LOG.info("Skipping non-MOB table [{}]", table);</span>
+<span class="source-line-no">083</span><span id="line-83"> return;</span>
+<span class="source-line-no">084</span><span id="line-84"> } else {</span>
+<span class="source-line-no">085</span><span id="line-85"> LOG.info("Only MOB files whose creation time older than {} will be archived, table={}",</span>
+<span class="source-line-no">086</span><span id="line-86"> new Date(maxCreationTimeToArchive), table);</span>
+<span class="source-line-no">087</span><span id="line-87"> }</span>
+<span class="source-line-no">088</span><span id="line-88"></span>
+<span class="source-line-no">089</span><span id="line-89"> FileSystem fs = FileSystem.get(conf);</span>
+<span class="source-line-no">090</span><span id="line-90"> Set<String> regionNames = new HashSet<>();</span>
+<span class="source-line-no">091</span><span id="line-91"> Path rootDir = CommonFSUtils.getRootDir(conf);</span>
+<span class="source-line-no">092</span><span id="line-92"> Path tableDir = CommonFSUtils.getTableDir(rootDir, table);</span>
+<span class="source-line-no">093</span><span id="line-93"> List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);</span>
+<span class="source-line-no">094</span><span id="line-94"></span>
+<span class="source-line-no">095</span><span id="line-95"> Set<String> allActiveMobFileName = new HashSet<String>();</span>
+<span class="source-line-no">096</span><span id="line-96"> for (Path regionPath : regionDirs) {</span>
+<span class="source-line-no">097</span><span id="line-97"> regionNames.add(regionPath.getName());</span>
+<span class="source-line-no">098</span><span id="line-98"> HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir,</span>
+<span class="source-line-no">099</span><span id="line-99"> HRegionFileSystem.loadRegionInfoFileContent(fs, regionPath));</span>
+<span class="source-line-no">100</span><span id="line-100"> for (ColumnFamilyDescriptor hcd : list) {</span>
+<span class="source-line-no">101</span><span id="line-101"> StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false);</span>
+<span class="source-line-no">102</span><span id="line-102"> String family = hcd.getNameAsString();</span>
+<span class="source-line-no">103</span><span id="line-103"> Path storePath = new Path(regionPath, family);</span>
+<span class="source-line-no">104</span><span id="line-104"> boolean succeed = false;</span>
+<span class="source-line-no">105</span><span id="line-105"> Set<String> regionMobs = new HashSet<String>();</span>
+<span class="source-line-no">106</span><span id="line-106"></span>
+<span class="source-line-no">107</span><span id="line-107"> while (!succeed) {</span>
+<span class="source-line-no">108</span><span id="line-108"> if (!fs.exists(storePath)) {</span>
+<span class="source-line-no">109</span><span id="line-109"> String errMsg = String.format("Directory %s was deleted during MOB file cleaner chore"</span>
+<span class="source-line-no">110</span><span id="line-110"> + " execution, aborting MOB file cleaner chore.", storePath);</span>
+<span class="source-line-no">111</span><span id="line-111"> throw new IOException(errMsg);</span>
+<span class="source-line-no">112</span><span id="line-112"> }</span>
+<span class="source-line-no">113</span><span id="line-113"> List<StoreFileInfo> storeFileInfos = sft.load();</span>
+<span class="source-line-no">114</span><span id="line-114"> LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath);</span>
+<span class="source-line-no">115</span><span id="line-115"> Path currentPath = null;</span>
+<span class="source-line-no">116</span><span id="line-116"> try {</span>
+<span class="source-line-no">117</span><span id="line-117"> for (StoreFileInfo storeFileInfo : storeFileInfos) {</span>
+<span class="source-line-no">118</span><span id="line-118"> Path pp = storeFileInfo.getPath();</span>
+<span class="source-line-no">119</span><span id="line-119"> currentPath = pp;</span>
+<span class="source-line-no">120</span><span id="line-120"> LOG.trace("Store file: {}", pp);</span>
+<span class="source-line-no">121</span><span id="line-121"> HStoreFile sf = null;</span>
+<span class="source-line-no">122</span><span id="line-122"> byte[] mobRefData = null;</span>
+<span class="source-line-no">123</span><span id="line-123"> byte[] bulkloadMarkerData = null;</span>
+<span class="source-line-no">124</span><span id="line-124"> try {</span>
+<span class="source-line-no">125</span><span id="line-125"> sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED);</span>
+<span class="source-line-no">126</span><span id="line-126"> sf.initReader();</span>
+<span class="source-line-no">127</span><span id="line-127"> mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS);</span>
+<span class="source-line-no">128</span><span id="line-128"> bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY);</span>
+<span class="source-line-no">129</span><span id="line-129"> // close store file to avoid memory leaks</span>
+<span class="source-line-no">130</span><span id="line-130"> sf.closeStoreFile(true);</span>
+<span class="source-line-no">131</span><span id="line-131"> } catch (IOException ex) {</span>
+<span class="source-line-no">132</span><span id="line-132"> // When FileBased SFT is active the store dir can contain corrupted or incomplete</span>
+<span class="source-line-no">133</span><span id="line-133"> // files. So read errors are expected. We just skip these files.</span>
+<span class="source-line-no">134</span><span id="line-134"> if (ex instanceof FileNotFoundException) {</span>
+<span class="source-line-no">135</span><span id="line-135"> throw ex;</span>
+<span class="source-line-no">136</span><span id="line-136"> }</span>
+<span class="source-line-no">137</span><span id="line-137"> LOG.debug("Failed to get mob data from file: {} due to error.", pp.toString(), ex);</span>
+<span class="source-line-no">138</span><span id="line-138"> continue;</span>
+<span class="source-line-no">139</span><span id="line-139"> }</span>
+<span class="source-line-no">140</span><span id="line-140"> if (mobRefData == null) {</span>
+<span class="source-line-no">141</span><span id="line-141"> if (bulkloadMarkerData == null) {</span>
+<span class="source-line-no">142</span><span id="line-142"> LOG.warn("Found old store file with no MOB_FILE_REFS: {} - "</span>
+<span class="source-line-no">143</span><span id="line-143"> + "can not proceed until all old files will be MOB-compacted.", pp);</span>
+<span class="source-line-no">144</span><span id="line-144"> return;</span>
+<span class="source-line-no">145</span><span id="line-145"> } else {</span>
+<span class="source-line-no">146</span><span id="line-146"> LOG.debug("Skipping file without MOB references (bulkloaded file):{}", pp);</span>
+<span class="source-line-no">147</span><span id="line-147"> continue;</span>
+<span class="source-line-no">148</span><span id="line-148"> }</span>
+<span class="source-line-no">149</span><span id="line-149"> }</span>
+<span class="source-line-no">150</span><span id="line-150"> // file may or may not have MOB references, but was created by the distributed</span>
+<span class="source-line-no">151</span><span id="line-151"> // mob compaction code.</span>
+<span class="source-line-no">152</span><span id="line-152"> try {</span>
+<span class="source-line-no">153</span><span id="line-153"> SetMultimap<TableName, String> mobs =</span>
+<span class="source-line-no">154</span><span id="line-154"> MobUtils.deserializeMobFileRefs(mobRefData).build();</span>
+<span class="source-line-no">155</span><span id="line-155"> LOG.debug("Found {} mob references for store={}", mobs.size(), sf);</span>
+<span class="source-line-no">156</span><span id="line-156"> LOG.trace("Specific mob references found for store={} : {}", sf, mobs);</span>
+<span class="source-line-no">157</span><span id="line-157"> regionMobs.addAll(mobs.values());</span>
+<span class="source-line-no">158</span><span id="line-158"> } catch (RuntimeException exception) {</span>
+<span class="source-line-no">159</span><span id="line-159"> throw new IOException("failure getting mob references for hfile " + sf, exception);</span>
+<span class="source-line-no">160</span><span id="line-160"> }</span>
+<span class="source-line-no">161</span><span id="line-161"> }</span>
+<span class="source-line-no">162</span><span id="line-162"> } catch (FileNotFoundException e) {</span>
+<span class="source-line-no">163</span><span id="line-163"> LOG.warn(</span>
+<span class="source-line-no">164</span><span id="line-164"> "Missing file:{} Starting MOB cleaning cycle from the beginning" + " due to error",</span>
+<span class="source-line-no">165</span><span id="line-165"> currentPath, e);</span>
+<span class="source-line-no">166</span><span id="line-166"> regionMobs.clear();</span>
+<span class="source-line-no">167</span><span id="line-167"> continue;</span>
+<span class="source-line-no">168</span><span id="line-168"> }</span>
+<span class="source-line-no">169</span><span id="line-169"> succeed = true;</span>
+<span class="source-line-no">170</span><span id="line-170"> }</span>
+<span class="source-line-no">171</span><span id="line-171"></span>
+<span class="source-line-no">172</span><span id="line-172"> // Add MOB references for current region/family</span>
+<span class="source-line-no">173</span><span id="line-173"> allActiveMobFileName.addAll(regionMobs);</span>
+<span class="source-line-no">174</span><span id="line-174"> } // END column families</span>
+<span class="source-line-no">175</span><span id="line-175"> } // END regions</span>
+<span class="source-line-no">176</span><span id="line-176"> // Check if number of MOB files too big (over 1M)</span>
+<span class="source-line-no">177</span><span id="line-177"> if (allActiveMobFileName.size() > 1000000) {</span>
+<span class="source-line-no">178</span><span id="line-178"> LOG.warn("Found too many active MOB files: {}, table={}, "</span>
+<span class="source-line-no">179</span><span id="line-179"> + "this may result in high memory pressure.", allActiveMobFileName.size(), table);</span>
+<span class="source-line-no">180</span><span id="line-180"> }</span>
+<span class="source-line-no">181</span><span id="line-181"> LOG.debug("Found: {} active mob refs for table={}", allActiveMobFileName.size(), table);</span>
+<span class="source-line-no">182</span><span id="line-182"> allActiveMobFileName.stream().forEach(LOG::trace);</span>
+<span class="source-line-no">183</span><span id="line-183"></span>
+<span class="source-line-no">184</span><span id="line-184"> // Now scan MOB directories and find MOB files with no references to them</span>
+<span class="source-line-no">185</span><span id="line-185"> for (ColumnFamilyDescriptor hcd : list) {</span>
+<span class="source-line-no">186</span><span id="line-186"> checkColumnFamilyDescriptor(conf, table, fs, admin, hcd, regionNames,</span>
+<span class="source-line-no">187</span><span id="line-187"> maxCreationTimeToArchive);</span>
+<span class="source-line-no">188</span><span id="line-188"> }</span>
+<span class="source-line-no">189</span><span id="line-189"> }</span>
+<span class="source-line-no">190</span><span id="line-190"></span>
+<span class="source-line-no">191</span><span id="line-191"> private static void checkColumnFamilyDescriptor(Configuration conf, TableName table,</span>
+<span class="source-line-no">192</span><span id="line-192"> FileSystem fs, Admin admin, ColumnFamilyDescriptor hcd, Set<String> regionNames,</span>
+<span class="source-line-no">193</span><span id="line-193"> long maxCreationTimeToArchive) throws IOException {</span>
+<span class="source-line-no">194</span><span id="line-194"> List<Path> toArchive = new ArrayList<Path>();</span>
+<span class="source-line-no">195</span><span id="line-195"> String family = hcd.getNameAsString();</span>
+<span class="source-line-no">196</span><span id="line-196"> Path dir = MobUtils.getMobFamilyPath(conf, table, family);</span>
+<span class="source-line-no">197</span><span id="line-197"> RemoteIterator<LocatedFileStatus> rit = fs.listLocatedStatus(dir);</span>
+<span class="source-line-no">198</span><span id="line-198"> while (rit.hasNext()) {</span>
+<span class="source-line-no">199</span><span id="line-199"> LocatedFileStatus lfs = rit.next();</span>
+<span class="source-line-no">200</span><span id="line-200"> Path p = lfs.getPath();</span>
+<span class="source-line-no">201</span><span id="line-201"> String[] mobParts = p.getName().split("_");</span>
+<span class="source-line-no">202</span><span id="line-202"> String regionName = mobParts[mobParts.length - 1];</span>
+<span class="source-line-no">203</span><span id="line-203"></span>
+<span class="source-line-no">204</span><span id="line-204"> if (!regionNames.contains(regionName)) {</span>
+<span class="source-line-no">205</span><span id="line-205"> // MOB belonged to a region no longer hosted</span>
+<span class="source-line-no">206</span><span id="line-206"> long creationTime = fs.getFileStatus(p).getModificationTime();</span>
+<span class="source-line-no">207</span><span id="line-207"> if (creationTime < maxCreationTimeToArchive) {</span>
+<span class="source-line-no">208</span><span id="line-208"> LOG.trace("Archiving MOB file {} creation time={}", p,</span>
+<span class="source-line-no">209</span><span id="line-209"> (fs.getFileStatus(p).getModificationTime()));</span>
+<span class="source-line-no">210</span><span id="line-210"> toArchive.add(p);</span>
+<span class="source-line-no">211</span><span id="line-211"> } else {</span>
+<span class="source-line-no">212</span><span id="line-212"> LOG.trace("Skipping fresh file: {}. Creation time={}", p,</span>
+<span class="source-line-no">213</span><span id="line-213"> fs.getFileStatus(p).getModificationTime());</span>
+<span class="source-line-no">214</span><span id="line-214"> }</span>
+<span class="source-line-no">215</span><span id="line-215"> } else {</span>
+<span class="source-line-no">216</span><span id="line-216"> LOG.trace("Keeping MOB file with existing region: {}", p);</span>
+<span class="source-line-no">217</span><span id="line-217"> }</span>
+<span class="source-line-no">218</span><span id="line-218"> }</span>
+<span class="source-line-no">219</span><span id="line-219"> LOG.info(" MOB Cleaner found {} files to archive for table={} family={}", toArchive.size(),</span>
+<span class="source-line-no">220</span><span id="line-220"> table, family);</span>
+<span class="source-line-no">221</span><span id="line-221"> archiveMobFiles(conf, table, admin, family.getBytes(), toArchive);</span>
+<span class="source-line-no">222</span><span id="line-222"> LOG.info(" MOB Cleaner archived {} files, table={} family={}", toArchive.size(), table, family);</span>
+<span class="source-line-no">223</span><span id="line-223"> }</span>
+<span class="source-line-no">224</span><span id="line-224"></span>
+<span class="source-line-no">225</span><span id="line-225"> /**</span>
+<span class="source-line-no">226</span><span id="line-226"> * Archives the mob files.</span>
+<span class="source-line-no">227</span><span id="line-227"> * @param conf The current configuration.</span>
+<span class="source-line-no">228</span><span id="line-228"> * @param tableName The table name.</span>
+<span class="source-line-no">229</span><span id="line-229"> * @param family The name of the column family.</span>
+<span class="source-line-no">230</span><span id="line-230"> * @param storeFiles The files to be archived.</span>
+<span class="source-line-no">231</span><span id="line-231"> * @throws IOException exception</span>
+<span class="source-line-no">232</span><span id="line-232"> */</span>
+<span class="source-line-no">233</span><span id="line-233"> private static void archiveMobFiles(Configuration conf, TableName tableName, Admin admin,</span>
+<span class="source-line-no">234</span><span id="line-234"> byte[] family, List<Path> storeFiles) throws IOException {</span>
+<span class="source-line-no">235</span><span id="line-235"></span>
+<span class="source-line-no">236</span><span id="line-236"> if (storeFiles.size() == 0) {</span>
+<span class="source-line-no">237</span><span id="line-237"> // nothing to remove</span>
+<span class="source-line-no">238</span><span id="line-238"> LOG.debug("Skipping archiving old MOB files - no files found for table={} cf={}", tableName,</span>
+<span class="source-line-no">239</span><span id="line-239"> Bytes.toString(family));</span>
+<span class="source-line-no">240</span><span id="line-240"> return;</span>
+<span class="source-line-no">241</span><span id="line-241"> }</span>
+<span class="source-line-no">242</span><span id="line-242"> Path mobTableDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);</span>
+<span class="source-line-no">243</span><span id="line-243"> FileSystem fs = storeFiles.get(0).getFileSystem(conf);</span>
+<span class="source-line-no">244</span><span id="line-244"></span>
+<span class="source-line-no">245</span><span id="line-245"> for (Path p : storeFiles) {</span>
+<span class="source-line-no">246</span><span id="line-246"> LOG.debug("MOB Cleaner is archiving: {}", p);</span>
+<span class="source-line-no">247</span><span id="line-247"> HFileArchiver.archiveStoreFile(conf, fs, MobUtils.getMobRegionInfo(tableName), mobTableDir,</span>
+<span class="source-line-no">248</span><span id="line-248"> family, p);</span>
+<span class="source-line-no">249</span><span id="line-249"> }</span>
+<span class="source-line-no">250</span><span id="line-250"> }</span>
+<span class="source-line-no">251</span><span id="line-251">}</span>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRequest.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRequest.html
index c5d993b34bc..41201564f78 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRequest.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRequest.html
@@ -604,24 +604,32 @@
<span class="source-line-no">591</span><span id="line-591"> + (3 * Bytes.SIZEOF_INT) + (4 * Bytes.SIZEOF_LONG));</span>
<span class="source-line-no">592</span><span id="line-592"></span>
<span class="source-line-no">593</span><span id="line-593"> /**</span>
-<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL.</span>
-<span class="source-line-no">595</span><span id="line-595"> */</span>
-<span class="source-line-no">596</span><span id="line-596"> @Override</span>
-<span class="source-line-no">597</span><span id="line-597"> DatanodeInfo[] getPipeline() {</span>
-<span class="source-line-no">598</span><span id="line-598"> if (this.hdfs_out != null) {</span>
-<span class="source-line-no">599</span><span id="line-599"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
-<span class="source-line-no">600</span><span id="line-600"> return ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
-<span class="source-line-no">601</span><span id="line-601"> }</span>
-<span class="source-line-no">602</span><span id="line-602"> }</span>
-<span class="source-line-no">603</span><span id="line-603"> return new DatanodeInfo[0];</span>
-<span class="source-line-no">604</span><span id="line-604"> }</span>
-<span class="source-line-no">605</span><span id="line-605"></span>
-<span class="source-line-no">606</span><span id="line-606"> @Override</span>
-<span class="source-line-no">607</span><span id="line-607"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
-<span class="source-line-no">608</span><span id="line-608"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
-<span class="source-line-no">609</span><span id="line-609"> return CombinedWriter.create(remoteWriter, localWriter);</span>
-<span class="source-line-no">610</span><span id="line-610"> }</span>
-<span class="source-line-no">611</span><span id="line-611">}</span>
+<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL. Note that DFSOutputStream#getPipeline() can</span>
+<span class="source-line-no">595</span><span id="line-595"> * legitimately return null (e.g. the underlying streamer is closed, or no block pipeline is</span>
+<span class="source-line-no">596</span><span id="line-596"> * currently established, such as between blocks) -- see HDFS-826. We normalize that to an empty</span>
+<span class="source-line-no">597</span><span id="line-597"> * array here so that callers (in particular debug-log formatting in AbstractFSWAL) never have to</span>
+<span class="source-line-no">598</span><span id="line-598"> * null-check, matching the contract already honored by AsyncFSWAL#getPipeline().</span>
+<span class="source-line-no">599</span><span id="line-599"> */</span>
+<span class="source-line-no">600</span><span id="line-600"> @Override</span>
+<span class="source-line-no">601</span><span id="line-601"> DatanodeInfo[] getPipeline() {</span>
+<span class="source-line-no">602</span><span id="line-602"> if (this.hdfs_out != null) {</span>
+<span class="source-line-no">603</span><span id="line-603"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
+<span class="source-line-no">604</span><span id="line-604"> DatanodeInfo[] pipeline =</span>
+<span class="source-line-no">605</span><span id="line-605"> ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
+<span class="source-line-no">606</span><span id="line-606"> if (pipeline != null) {</span>
+<span class="source-line-no">607</span><span id="line-607"> return pipeline;</span>
+<span class="source-line-no">608</span><span id="line-608"> }</span>
+<span class="source-line-no">609</span><span id="line-609"> }</span>
+<span class="source-line-no">610</span><span id="line-610"> }</span>
+<span class="source-line-no">611</span><span id="line-611"> return new DatanodeInfo[0];</span>
+<span class="source-line-no">612</span><span id="line-612"> }</span>
+<span class="source-line-no">613</span><span id="line-613"></span>
+<span class="source-line-no">614</span><span id="line-614"> @Override</span>
+<span class="source-line-no">615</span><span id="line-615"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
+<span class="source-line-no">616</span><span id="line-616"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
+<span class="source-line-no">617</span><span id="line-617"> return CombinedWriter.create(remoteWriter, localWriter);</span>
+<span class="source-line-no">618</span><span id="line-618"> }</span>
+<span class="source-line-no">619</span><span id="line-619">}</span>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html
index e5f9638fc62..d9f948c9017 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html
@@ -604,24 +604,32 @@
<span class="source-line-no">591</span><span id="line-591"> + (3 * Bytes.SIZEOF_INT) + (4 * Bytes.SIZEOF_LONG));</span>
<span class="source-line-no">592</span><span id="line-592"></span>
<span class="source-line-no">593</span><span id="line-593"> /**</span>
-<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL.</span>
-<span class="source-line-no">595</span><span id="line-595"> */</span>
-<span class="source-line-no">596</span><span id="line-596"> @Override</span>
-<span class="source-line-no">597</span><span id="line-597"> DatanodeInfo[] getPipeline() {</span>
-<span class="source-line-no">598</span><span id="line-598"> if (this.hdfs_out != null) {</span>
-<span class="source-line-no">599</span><span id="line-599"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
-<span class="source-line-no">600</span><span id="line-600"> return ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
-<span class="source-line-no">601</span><span id="line-601"> }</span>
-<span class="source-line-no">602</span><span id="line-602"> }</span>
-<span class="source-line-no">603</span><span id="line-603"> return new DatanodeInfo[0];</span>
-<span class="source-line-no">604</span><span id="line-604"> }</span>
-<span class="source-line-no">605</span><span id="line-605"></span>
-<span class="source-line-no">606</span><span id="line-606"> @Override</span>
-<span class="source-line-no">607</span><span id="line-607"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
-<span class="source-line-no">608</span><span id="line-608"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
-<span class="source-line-no">609</span><span id="line-609"> return CombinedWriter.create(remoteWriter, localWriter);</span>
-<span class="source-line-no">610</span><span id="line-610"> }</span>
-<span class="source-line-no">611</span><span id="line-611">}</span>
+<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL. Note that DFSOutputStream#getPipeline() can</span>
+<span class="source-line-no">595</span><span id="line-595"> * legitimately return null (e.g. the underlying streamer is closed, or no block pipeline is</span>
+<span class="source-line-no">596</span><span id="line-596"> * currently established, such as between blocks) -- see HDFS-826. We normalize that to an empty</span>
+<span class="source-line-no">597</span><span id="line-597"> * array here so that callers (in particular debug-log formatting in AbstractFSWAL) never have to</span>
+<span class="source-line-no">598</span><span id="line-598"> * null-check, matching the contract already honored by AsyncFSWAL#getPipeline().</span>
+<span class="source-line-no">599</span><span id="line-599"> */</span>
+<span class="source-line-no">600</span><span id="line-600"> @Override</span>
+<span class="source-line-no">601</span><span id="line-601"> DatanodeInfo[] getPipeline() {</span>
+<span class="source-line-no">602</span><span id="line-602"> if (this.hdfs_out != null) {</span>
+<span class="source-line-no">603</span><span id="line-603"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
+<span class="source-line-no">604</span><span id="line-604"> DatanodeInfo[] pipeline =</span>
+<span class="source-line-no">605</span><span id="line-605"> ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
+<span class="source-line-no">606</span><span id="line-606"> if (pipeline != null) {</span>
+<span class="source-line-no">607</span><span id="line-607"> return pipeline;</span>
+<span class="source-line-no">608</span><span id="line-608"> }</span>
+<span class="source-line-no">609</span><span id="line-609"> }</span>
+<span class="source-line-no">610</span><span id="line-610"> }</span>
+<span class="source-line-no">611</span><span id="line-611"> return new DatanodeInfo[0];</span>
+<span class="source-line-no">612</span><span id="line-612"> }</span>
+<span class="source-line-no">613</span><span id="line-613"></span>
+<span class="source-line-no">614</span><span id="line-614"> @Override</span>
+<span class="source-line-no">615</span><span id="line-615"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
+<span class="source-line-no">616</span><span id="line-616"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
+<span class="source-line-no">617</span><span id="line-617"> return CombinedWriter.create(remoteWriter, localWriter);</span>
+<span class="source-line-no">618</span><span id="line-618"> }</span>
+<span class="source-line-no">619</span><span id="line-619">}</span>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
index d0c93fa6d3c..b660f6a4e16 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html
@@ -604,24 +604,32 @@
<span class="source-line-no">591</span><span id="line-591"> + (3 * Bytes.SIZEOF_INT) + (4 * Bytes.SIZEOF_LONG));</span>
<span class="source-line-no">592</span><span id="line-592"></span>
<span class="source-line-no">593</span><span id="line-593"> /**</span>
-<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL.</span>
-<span class="source-line-no">595</span><span id="line-595"> */</span>
-<span class="source-line-no">596</span><span id="line-596"> @Override</span>
-<span class="source-line-no">597</span><span id="line-597"> DatanodeInfo[] getPipeline() {</span>
-<span class="source-line-no">598</span><span id="line-598"> if (this.hdfs_out != null) {</span>
-<span class="source-line-no">599</span><span id="line-599"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
-<span class="source-line-no">600</span><span id="line-600"> return ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
-<span class="source-line-no">601</span><span id="line-601"> }</span>
-<span class="source-line-no">602</span><span id="line-602"> }</span>
-<span class="source-line-no">603</span><span id="line-603"> return new DatanodeInfo[0];</span>
-<span class="source-line-no">604</span><span id="line-604"> }</span>
-<span class="source-line-no">605</span><span id="line-605"></span>
-<span class="source-line-no">606</span><span id="line-606"> @Override</span>
-<span class="source-line-no">607</span><span id="line-607"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
-<span class="source-line-no">608</span><span id="line-608"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
-<span class="source-line-no">609</span><span id="line-609"> return CombinedWriter.create(remoteWriter, localWriter);</span>
-<span class="source-line-no">610</span><span id="line-610"> }</span>
-<span class="source-line-no">611</span><span id="line-611">}</span>
+<span class="source-line-no">594</span><span id="line-594"> * This method gets the pipeline for the current WAL. Note that DFSOutputStream#getPipeline() can</span>
+<span class="source-line-no">595</span><span id="line-595"> * legitimately return null (e.g. the underlying streamer is closed, or no block pipeline is</span>
+<span class="source-line-no">596</span><span id="line-596"> * currently established, such as between blocks) -- see HDFS-826. We normalize that to an empty</span>
+<span class="source-line-no">597</span><span id="line-597"> * array here so that callers (in particular debug-log formatting in AbstractFSWAL) never have to</span>
+<span class="source-line-no">598</span><span id="line-598"> * null-check, matching the contract already honored by AsyncFSWAL#getPipeline().</span>
+<span class="source-line-no">599</span><span id="line-599"> */</span>
+<span class="source-line-no">600</span><span id="line-600"> @Override</span>
+<span class="source-line-no">601</span><span id="line-601"> DatanodeInfo[] getPipeline() {</span>
+<span class="source-line-no">602</span><span id="line-602"> if (this.hdfs_out != null) {</span>
+<span class="source-line-no">603</span><span id="line-603"> if (this.hdfs_out.getWrappedStream() instanceof DFSOutputStream) {</span>
+<span class="source-line-no">604</span><span id="line-604"> DatanodeInfo[] pipeline =</span>
+<span class="source-line-no">605</span><span id="line-605"> ((DFSOutputStream) this.hdfs_out.getWrappedStream()).getPipeline();</span>
+<span class="source-line-no">606</span><span id="line-606"> if (pipeline != null) {</span>
+<span class="source-line-no">607</span><span id="line-607"> return pipeline;</span>
+<span class="source-line-no">608</span><span id="line-608"> }</span>
+<span class="source-line-no">609</span><span id="line-609"> }</span>
+<span class="source-line-no">610</span><span id="line-610"> }</span>
+<span class="source-line-no">611</span><span id="line-611"> return new DatanodeInfo[0];</span>
+<span class="source-line-no">612</span><span id="line-612"> }</span>
+<span class="source-line-no">613</span><span id="line-613"></span>
+<span class="source-line-no">614</span><span id="line-614"> @Override</span>
+<span class="source-line-no">615</span><span id="line-615"> protected Writer createCombinedWriter(Writer localWriter, Writer remoteWriter) {</span>
+<span class="source-line-no">616</span><span id="line-616"> // put remote writer first as usually it will cost more time to finish, so we write to it first</span>
+<span class="source-line-no">617</span><span id="line-617"> return CombinedWriter.create(remoteWriter, localWriter);</span>
+<span class="source-line-no">618</span><span id="line-618"> }</span>
+<span class="source-line-no">619</span><span id="line-619">}</span>
diff --git a/hbase-website/dependency-convergence.html b/hbase-website/dependency-convergence.html
index b107959b9f3..ae5d9962b81 100644
--- a/hbase-website/dependency-convergence.html
+++ b/hbase-website/dependency-convergence.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-convergence at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-convergence at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/dependency-info.html b/hbase-website/dependency-info.html
index 4fd080fda2f..b682e82821e 100644
--- a/hbase-website/dependency-info.html
+++ b/hbase-website/dependency-info.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-info at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-info at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/dependency-management.html b/hbase-website/dependency-management.html
index 67536348434..8b21cfdb58c 100644
--- a/hbase-website/dependency-management.html
+++ b/hbase-website/dependency-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:dependency-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/index.html b/hbase-website/index.html
index 029d4814aa2..8ee5f8da369 100644
--- a/hbase-website/index.html
+++ b/hbase-website/index.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:index at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:index at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/issue-management.html b/hbase-website/issue-management.html
index 7075722aae2..162ca3a174c 100644
--- a/hbase-website/issue-management.html
+++ b/hbase-website/issue-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:issue-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:issue-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/licenses.html b/hbase-website/licenses.html
index 27a83811f49..e8e693e52ee 100644
--- a/hbase-website/licenses.html
+++ b/hbase-website/licenses.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:licenses at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:licenses at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/mailing-lists.html b/hbase-website/mailing-lists.html
index 898e906adb4..51b98335d56 100644
--- a/hbase-website/mailing-lists.html
+++ b/hbase-website/mailing-lists.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:mailing-lists at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:mailing-lists at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/plugin-management.html b/hbase-website/plugin-management.html
index 643a8fef9d2..ce915e49061 100644
--- a/hbase-website/plugin-management.html
+++ b/hbase-website/plugin-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugin-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugin-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/plugins.html b/hbase-website/plugins.html
index 86c195c41f7..a511a21a972 100644
--- a/hbase-website/plugins.html
+++ b/hbase-website/plugins.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugins at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugins at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/project-info.html b/hbase-website/project-info.html
index 6c70e26af3f..4df72cfb56e 100644
--- a/hbase-website/project-info.html
+++ b/hbase-website/project-info.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-info at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-info at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/scm.html b/hbase-website/scm.html
index 70c447fa6dd..7a10a023afc 100644
--- a/hbase-website/scm.html
+++ b/hbase-website/scm.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:scm at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:scm at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/summary.html b/hbase-website/summary.html
index 3ac482b351c..0b5d618005c 100644
--- a/hbase-website/summary.html
+++ b/hbase-website/summary.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:summary at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:summary at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/hbase-website/team.html b/hbase-website/team.html
index bde0a49735b..47703bd64d3 100644
--- a/hbase-website/team.html
+++ b/hbase-website/team.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:team at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:team at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/issue-management.html b/issue-management.html
index b995c2bcf45..9db809109ba 100644
--- a/issue-management.html
+++ b/issue-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:issue-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:issue-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/licenses.html b/licenses.html
index 3d4f2b4a00e..3798a603450 100644
--- a/licenses.html
+++ b/licenses.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:licenses at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:licenses at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/mailing-lists.html b/mailing-lists.html
index 74be2cbf581..539368d407e 100644
--- a/mailing-lists.html
+++ b/mailing-lists.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:mailing-lists at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:mailing-lists at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/plugin-management.html b/plugin-management.html
index 017f1b9dd54..b5b066c26d3 100644
--- a/plugin-management.html
+++ b/plugin-management.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugin-management at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugin-management at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/plugins.html b/plugins.html
index c46a29c877d..cbf6445e9ba 100644
--- a/plugins.html
+++ b/plugins.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugins at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:plugins at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/project-info.html b/project-info.html
index ff54bcf080c..8a1d05c1203 100644
--- a/project-info.html
+++ b/project-info.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-info at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-info at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/project-reports.html b/project-reports.html
index 3499654a87c..a0f18c07578 100644
--- a/project-reports.html
+++ b/project-reports.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-reports at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-site-plugin:3.21.0:project-reports at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/scm.html b/scm.html
index a1bcf6f6f60..693ac394736 100644
--- a/scm.html
+++ b/scm.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:scm at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:scm at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/summary.html b/summary.html
index 46a860c615b..68fbeaee50a 100644
--- a/summary.html
+++ b/summary.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:summary at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:summary at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/team.html b/team.html
index 0fafeb43c80..b1695fac440 100644
--- a/team.html
+++ b/team.html
@@ -2,7 +2,7 @@
<!--
- | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:team at 2026-09-02
+ | Generated by Apache Maven Doxia Site Renderer 2.0.0 from org.apache.maven.plugins:maven-project-info-reports-plugin:3.9.0:team at 2026-09-03
| Rendered using Apache Maven Fluido Skin 2.0.0-M9
-->
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
diff --git a/testapidocs/index-all.html b/testapidocs/index-all.html
index c00680538af..dc81562c146 100644
--- a/testapidocs/index-all.html
+++ b/testapidocs/index-all.html
@@ -29820,6 +29820,11 @@ loadScripts(document, 'script');</script>
<dd> </dd>
<dt><a href="../apidocs/org/apache/hadoop/hbase/client/TestConnectionUtils.html#testGetPauseTime()" class="member-name-link external-link" title="class or interface in org.apache.hadoop.hbase.client">testGetPauseTime()</a> - Method in class org.apache.hadoop.hbase.client.<a href="../apidocs/org/apache/hadoop/hbase/client/TestConnectionUtils.html" title="class or interface in org.apache.hadoop.hbase.client" class="external-link">TestConnectionUtils</a></dt>
<dd> </dd>
+<dt><a href="../apidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()" class="member-name-link external-link" title="class or interface in org.apache.hadoop.hbase.regionserver.wal">testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()</a> - Method in class org.apache.hadoop.hbase.regionserver.wal.<a href="../apidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html" title="class or interface in or [...]
+<dd>
+<div class="block">Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the
+ underlying DFSOutputStream#getPipeline() legitimately returns null (e.g.</div>
+</dd>
<dt><a href="../apidocs/org/apache/hadoop/hbase/nio/TestMultiByteBuff.html#testGetPrimitivesWithSmallIndividualBBs()" class="member-name-link external-link" title="class or interface in org.apache.hadoop.hbase.nio">testGetPrimitivesWithSmallIndividualBBs()</a> - Method in class org.apache.hadoop.hbase.nio.<a href="../apidocs/org/apache/hadoop/hbase/nio/TestMultiByteBuff.html" title="class or interface in org.apache.hadoop.hbase.nio" class="external-link">TestMultiByteBuff</a></dt>
<dd> </dd>
<dt><a href="../apidocs/org/apache/hadoop/hbase/client/TestGetProcedureResult.html" class="type-name-link external-link" title="class or interface in org.apache.hadoop.hbase.client">TestGetProcedureResult</a> - Class in <a href="org/apache/hadoop/hbase/client/package-summary.html">org.apache.hadoop.hbase.client</a></dt>
diff --git a/testapidocs/member-search-index.js b/testapidocs/member-search-index.js
index 117d72aab64..2ce794dd574 100644
--- a/testapidocs/member-search-index.js
+++ b/testapidocs/member-search-index.js
@@ -1 +1 @@
-memberSearchIndex = [{"p":"org.apache.hadoop.hbase.backup","c":"TestSystemTableSnapshot","l":"_testBackupRestoreSystemTable()"},{"p":"org.apache.hadoop.hbase.coprocessor","c":"TestRegionObserverInterface","l":"A"},{"p":"org.apache.hadoop.hbase.client","c":"TestGetProcedureResult.DummyProcedure","l":"abort(MasterProcedureEnv)","u":"abort(org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv)"},{"p":"org.apache.hadoop.hbase.master","c":"TestMigrateNamespaceTable.SuspendProcedure","l" [...]
\ No newline at end of file
+memberSearchIndex = [{"p":"org.apache.hadoop.hbase.backup","c":"TestSystemTableSnapshot","l":"_testBackupRestoreSystemTable()"},{"p":"org.apache.hadoop.hbase.coprocessor","c":"TestRegionObserverInterface","l":"A"},{"p":"org.apache.hadoop.hbase.client","c":"TestGetProcedureResult.DummyProcedure","l":"abort(MasterProcedureEnv)","u":"abort(org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv)"},{"p":"org.apache.hadoop.hbase.master","c":"TestMigrateNamespaceTable.SuspendProcedure","l" [...]
\ No newline at end of file
diff --git a/testapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html b/testapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
index 0704cea1e9c..d387103699d 100644
--- a/testapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
+++ b/testapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
@@ -31,204 +31,244 @@
<span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.regionserver.wal;</span>
<span class="source-line-no">019</span><span id="line-19"></span>
<span class="source-line-no">020</span><span id="line-20">import static org.junit.jupiter.api.Assertions.assertEquals;</span>
-<span class="source-line-no">021</span><span id="line-21"></span>
-<span class="source-line-no">022</span><span id="line-22">import java.io.IOException;</span>
-<span class="source-line-no">023</span><span id="line-23">import java.lang.reflect.Field;</span>
-<span class="source-line-no">024</span><span id="line-24">import java.util.List;</span>
-<span class="source-line-no">025</span><span id="line-25">import java.util.NavigableMap;</span>
-<span class="source-line-no">026</span><span id="line-26">import java.util.TreeMap;</span>
-<span class="source-line-no">027</span><span id="line-27">import java.util.concurrent.CountDownLatch;</span>
-<span class="source-line-no">028</span><span id="line-28">import java.util.concurrent.ExecutorService;</span>
-<span class="source-line-no">029</span><span id="line-29">import java.util.concurrent.Executors;</span>
-<span class="source-line-no">030</span><span id="line-30">import java.util.concurrent.atomic.AtomicBoolean;</span>
-<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.conf.Configuration;</span>
-<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.fs.FileSystem;</span>
-<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.fs.Path;</span>
-<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.HConstants;</span>
-<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.TableName;</span>
-<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span>
-<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.client.Put;</span>
-<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.client.RegionInfo;</span>
-<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span>
-<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
-<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span>
-<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.ChunkCreator;</span>
-<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.regionserver.HRegion;</span>
-<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.regionserver.MemStoreLAB;</span>
-<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;</span>
-<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.testclassification.MediumTests;</span>
-<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.testclassification.RegionServerTests;</span>
-<span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.util.Bytes;</span>
-<span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
-<span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.util.Threads;</span>
-<span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.wal.WALEdit;</span>
-<span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.wal.WALKey;</span>
-<span class="source-line-no">053</span><span id="line-53">import org.junit.jupiter.api.BeforeEach;</span>
-<span class="source-line-no">054</span><span id="line-54">import org.junit.jupiter.api.Tag;</span>
-<span class="source-line-no">055</span><span id="line-55">import org.junit.jupiter.api.Test;</span>
-<span class="source-line-no">056</span><span id="line-56">import org.junit.jupiter.api.TestInfo;</span>
-<span class="source-line-no">057</span><span id="line-57"></span>
-<span class="source-line-no">058</span><span id="line-58">/**</span>
-<span class="source-line-no">059</span><span id="line-59"> * Provides FSHLog test cases.</span>
-<span class="source-line-no">060</span><span id="line-60"> */</span>
-<span class="source-line-no">061</span><span id="line-61">@Tag(RegionServerTests.TAG)</span>
-<span class="source-line-no">062</span><span id="line-62">@Tag(MediumTests.TAG)</span>
-<span class="source-line-no">063</span><span id="line-63">public class TestFSHLog extends AbstractTestFSWAL {</span>
+<span class="source-line-no">021</span><span id="line-21">import static org.junit.jupiter.api.Assertions.assertNotNull;</span>
+<span class="source-line-no">022</span><span id="line-22">import static org.mockito.Mockito.mock;</span>
+<span class="source-line-no">023</span><span id="line-23">import static org.mockito.Mockito.when;</span>
+<span class="source-line-no">024</span><span id="line-24"></span>
+<span class="source-line-no">025</span><span id="line-25">import java.io.IOException;</span>
+<span class="source-line-no">026</span><span id="line-26">import java.lang.reflect.Field;</span>
+<span class="source-line-no">027</span><span id="line-27">import java.util.List;</span>
+<span class="source-line-no">028</span><span id="line-28">import java.util.NavigableMap;</span>
+<span class="source-line-no">029</span><span id="line-29">import java.util.TreeMap;</span>
+<span class="source-line-no">030</span><span id="line-30">import java.util.concurrent.CountDownLatch;</span>
+<span class="source-line-no">031</span><span id="line-31">import java.util.concurrent.ExecutorService;</span>
+<span class="source-line-no">032</span><span id="line-32">import java.util.concurrent.Executors;</span>
+<span class="source-line-no">033</span><span id="line-33">import java.util.concurrent.atomic.AtomicBoolean;</span>
+<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.conf.Configuration;</span>
+<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.fs.FSDataOutputStream;</span>
+<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.fs.FileSystem;</span>
+<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.fs.Path;</span>
+<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.HConstants;</span>
+<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.TableName;</span>
+<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span>
+<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.client.Put;</span>
+<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.client.RegionInfo;</span>
+<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span>
+<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
+<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span>
+<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.regionserver.ChunkCreator;</span>
+<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.regionserver.HRegion;</span>
+<span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.regionserver.MemStoreLAB;</span>
+<span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;</span>
+<span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.testclassification.MediumTests;</span>
+<span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.testclassification.RegionServerTests;</span>
+<span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.util.Bytes;</span>
+<span class="source-line-no">053</span><span id="line-53">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
+<span class="source-line-no">054</span><span id="line-54">import org.apache.hadoop.hbase.util.Threads;</span>
+<span class="source-line-no">055</span><span id="line-55">import org.apache.hadoop.hbase.wal.WALEdit;</span>
+<span class="source-line-no">056</span><span id="line-56">import org.apache.hadoop.hbase.wal.WALKey;</span>
+<span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.hdfs.DFSOutputStream;</span>
+<span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;</span>
+<span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.hdfs.protocol.DatanodeInfo;</span>
+<span class="source-line-no">060</span><span id="line-60">import org.junit.jupiter.api.BeforeEach;</span>
+<span class="source-line-no">061</span><span id="line-61">import org.junit.jupiter.api.Tag;</span>
+<span class="source-line-no">062</span><span id="line-62">import org.junit.jupiter.api.Test;</span>
+<span class="source-line-no">063</span><span id="line-63">import org.junit.jupiter.api.TestInfo;</span>
<span class="source-line-no">064</span><span id="line-64"></span>
-<span class="source-line-no">065</span><span id="line-65"> private String name;</span>
-<span class="source-line-no">066</span><span id="line-66"></span>
-<span class="source-line-no">067</span><span id="line-67"> @BeforeEach</span>
-<span class="source-line-no">068</span><span id="line-68"> public void initTestName(TestInfo testInfo) {</span>
-<span class="source-line-no">069</span><span id="line-69"> name = testInfo.getTestMethod().get().getName();</span>
-<span class="source-line-no">070</span><span id="line-70"> }</span>
+<span class="source-line-no">065</span><span id="line-65">/**</span>
+<span class="source-line-no">066</span><span id="line-66"> * Provides FSHLog test cases.</span>
+<span class="source-line-no">067</span><span id="line-67"> */</span>
+<span class="source-line-no">068</span><span id="line-68">@Tag(RegionServerTests.TAG)</span>
+<span class="source-line-no">069</span><span id="line-69">@Tag(MediumTests.TAG)</span>
+<span class="source-line-no">070</span><span id="line-70">public class TestFSHLog extends AbstractTestFSWAL {</span>
<span class="source-line-no">071</span><span id="line-71"></span>
-<span class="source-line-no">072</span><span id="line-72"> @Override</span>
-<span class="source-line-no">073</span><span id="line-73"> protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String walDir, String archiveDir,</span>
-<span class="source-line-no">074</span><span id="line-74"> Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix,</span>
-<span class="source-line-no">075</span><span id="line-75"> String suffix) throws IOException {</span>
-<span class="source-line-no">076</span><span id="line-76"> FSHLog wal =</span>
-<span class="source-line-no">077</span><span id="line-77"> new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);</span>
-<span class="source-line-no">078</span><span id="line-78"> wal.init();</span>
-<span class="source-line-no">079</span><span id="line-79"> return wal;</span>
-<span class="source-line-no">080</span><span id="line-80"> }</span>
-<span class="source-line-no">081</span><span id="line-81"></span>
-<span class="source-line-no">082</span><span id="line-82"> @Override</span>
-<span class="source-line-no">083</span><span id="line-83"> protected AbstractFSWAL<?> newSlowWAL(FileSystem fs, Path rootDir, String walDir,</span>
-<span class="source-line-no">084</span><span id="line-84"> String archiveDir, Configuration conf, List<WALActionsListener> listeners,</span>
-<span class="source-line-no">085</span><span id="line-85"> boolean failIfWALExists, String prefix, String suffix, final Runnable action)</span>
-<span class="source-line-no">086</span><span id="line-86"> throws IOException {</span>
-<span class="source-line-no">087</span><span id="line-87"> FSHLog wal = new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists,</span>
-<span class="source-line-no">088</span><span id="line-88"> prefix, suffix) {</span>
-<span class="source-line-no">089</span><span id="line-89"></span>
-<span class="source-line-no">090</span><span id="line-90"> @Override</span>
-<span class="source-line-no">091</span><span id="line-91"> protected void atHeadOfRingBufferEventHandlerAppend() {</span>
-<span class="source-line-no">092</span><span id="line-92"> action.run();</span>
-<span class="source-line-no">093</span><span id="line-93"> super.atHeadOfRingBufferEventHandlerAppend();</span>
-<span class="source-line-no">094</span><span id="line-94"> }</span>
-<span class="source-line-no">095</span><span id="line-95"> };</span>
-<span class="source-line-no">096</span><span id="line-96"> wal.init();</span>
-<span class="source-line-no">097</span><span id="line-97"> return wal;</span>
-<span class="source-line-no">098</span><span id="line-98"> }</span>
-<span class="source-line-no">099</span><span id="line-99"></span>
-<span class="source-line-no">100</span><span id="line-100"> @Test</span>
-<span class="source-line-no">101</span><span id="line-101"> public void testSyncRunnerIndexOverflow() throws IOException, NoSuchFieldException,</span>
-<span class="source-line-no">102</span><span id="line-102"> SecurityException, IllegalArgumentException, IllegalAccessException {</span>
-<span class="source-line-no">103</span><span id="line-103"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
-<span class="source-line-no">104</span><span id="line-104"> FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
-<span class="source-line-no">105</span><span id="line-105"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);</span>
-<span class="source-line-no">106</span><span id="line-106"> log.init();</span>
-<span class="source-line-no">107</span><span id="line-107"> try {</span>
-<span class="source-line-no">108</span><span id="line-108"> Field syncRunnerIndexField = FSHLog.class.getDeclaredField("syncRunnerIndex");</span>
-<span class="source-line-no">109</span><span id="line-109"> syncRunnerIndexField.setAccessible(true);</span>
-<span class="source-line-no">110</span><span id="line-110"> syncRunnerIndexField.set(log, Integer.MAX_VALUE - 1);</span>
-<span class="source-line-no">111</span><span id="line-111"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
-<span class="source-line-no">112</span><span id="line-112"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();</span>
-<span class="source-line-no">113</span><span id="line-113"> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span>
-<span class="source-line-no">114</span><span id="line-114"> for (byte[] fam : htd.getColumnFamilyNames()) {</span>
-<span class="source-line-no">115</span><span id="line-115"> scopes.put(fam, 0);</span>
-<span class="source-line-no">116</span><span id="line-116"> }</span>
-<span class="source-line-no">117</span><span id="line-117"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
-<span class="source-line-no">118</span><span id="line-118"> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();</span>
-<span class="source-line-no">119</span><span id="line-119"> for (int i = 0; i < 10; i++) {</span>
-<span class="source-line-no">120</span><span id="line-120"> addEdits(log, hri, htd, 1, mvcc, scopes, "row");</span>
-<span class="source-line-no">121</span><span id="line-121"> }</span>
-<span class="source-line-no">122</span><span id="line-122"> } finally {</span>
-<span class="source-line-no">123</span><span id="line-123"> log.close();</span>
-<span class="source-line-no">124</span><span id="line-124"> }</span>
-<span class="source-line-no">125</span><span id="line-125"> }</span>
-<span class="source-line-no">126</span><span id="line-126"></span>
-<span class="source-line-no">127</span><span id="line-127"> /**</span>
-<span class="source-line-no">128</span><span id="line-128"> * Test case for https://issues.apache.org/jira/browse/HBASE-16721</span>
-<span class="source-line-no">129</span><span id="line-129"> */</span>
-<span class="source-line-no">130</span><span id="line-130"> @Test</span>
-<span class="source-line-no">131</span><span id="line-131"> public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {</span>
-<span class="source-line-no">132</span><span id="line-132"> final byte[] b = Bytes.toBytes("b");</span>
+<span class="source-line-no">072</span><span id="line-72"> private String name;</span>
+<span class="source-line-no">073</span><span id="line-73"></span>
+<span class="source-line-no">074</span><span id="line-74"> @BeforeEach</span>
+<span class="source-line-no">075</span><span id="line-75"> public void initTestName(TestInfo testInfo) {</span>
+<span class="source-line-no">076</span><span id="line-76"> name = testInfo.getTestMethod().get().getName();</span>
+<span class="source-line-no">077</span><span id="line-77"> }</span>
+<span class="source-line-no">078</span><span id="line-78"></span>
+<span class="source-line-no">079</span><span id="line-79"> @Override</span>
+<span class="source-line-no">080</span><span id="line-80"> protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String walDir, String archiveDir,</span>
+<span class="source-line-no">081</span><span id="line-81"> Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix,</span>
+<span class="source-line-no">082</span><span id="line-82"> String suffix) throws IOException {</span>
+<span class="source-line-no">083</span><span id="line-83"> FSHLog wal =</span>
+<span class="source-line-no">084</span><span id="line-84"> new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);</span>
+<span class="source-line-no">085</span><span id="line-85"> wal.init();</span>
+<span class="source-line-no">086</span><span id="line-86"> return wal;</span>
+<span class="source-line-no">087</span><span id="line-87"> }</span>
+<span class="source-line-no">088</span><span id="line-88"></span>
+<span class="source-line-no">089</span><span id="line-89"> @Override</span>
+<span class="source-line-no">090</span><span id="line-90"> protected AbstractFSWAL<?> newSlowWAL(FileSystem fs, Path rootDir, String walDir,</span>
+<span class="source-line-no">091</span><span id="line-91"> String archiveDir, Configuration conf, List<WALActionsListener> listeners,</span>
+<span class="source-line-no">092</span><span id="line-92"> boolean failIfWALExists, String prefix, String suffix, final Runnable action)</span>
+<span class="source-line-no">093</span><span id="line-93"> throws IOException {</span>
+<span class="source-line-no">094</span><span id="line-94"> FSHLog wal = new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists,</span>
+<span class="source-line-no">095</span><span id="line-95"> prefix, suffix) {</span>
+<span class="source-line-no">096</span><span id="line-96"></span>
+<span class="source-line-no">097</span><span id="line-97"> @Override</span>
+<span class="source-line-no">098</span><span id="line-98"> protected void atHeadOfRingBufferEventHandlerAppend() {</span>
+<span class="source-line-no">099</span><span id="line-99"> action.run();</span>
+<span class="source-line-no">100</span><span id="line-100"> super.atHeadOfRingBufferEventHandlerAppend();</span>
+<span class="source-line-no">101</span><span id="line-101"> }</span>
+<span class="source-line-no">102</span><span id="line-102"> };</span>
+<span class="source-line-no">103</span><span id="line-103"> wal.init();</span>
+<span class="source-line-no">104</span><span id="line-104"> return wal;</span>
+<span class="source-line-no">105</span><span id="line-105"> }</span>
+<span class="source-line-no">106</span><span id="line-106"></span>
+<span class="source-line-no">107</span><span id="line-107"> @Test</span>
+<span class="source-line-no">108</span><span id="line-108"> public void testSyncRunnerIndexOverflow() throws IOException, NoSuchFieldException,</span>
+<span class="source-line-no">109</span><span id="line-109"> SecurityException, IllegalArgumentException, IllegalAccessException {</span>
+<span class="source-line-no">110</span><span id="line-110"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">111</span><span id="line-111"> FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">112</span><span id="line-112"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);</span>
+<span class="source-line-no">113</span><span id="line-113"> log.init();</span>
+<span class="source-line-no">114</span><span id="line-114"> try {</span>
+<span class="source-line-no">115</span><span id="line-115"> Field syncRunnerIndexField = FSHLog.class.getDeclaredField("syncRunnerIndex");</span>
+<span class="source-line-no">116</span><span id="line-116"> syncRunnerIndexField.setAccessible(true);</span>
+<span class="source-line-no">117</span><span id="line-117"> syncRunnerIndexField.set(log, Integer.MAX_VALUE - 1);</span>
+<span class="source-line-no">118</span><span id="line-118"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
+<span class="source-line-no">119</span><span id="line-119"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();</span>
+<span class="source-line-no">120</span><span id="line-120"> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span>
+<span class="source-line-no">121</span><span id="line-121"> for (byte[] fam : htd.getColumnFamilyNames()) {</span>
+<span class="source-line-no">122</span><span id="line-122"> scopes.put(fam, 0);</span>
+<span class="source-line-no">123</span><span id="line-123"> }</span>
+<span class="source-line-no">124</span><span id="line-124"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
+<span class="source-line-no">125</span><span id="line-125"> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();</span>
+<span class="source-line-no">126</span><span id="line-126"> for (int i = 0; i < 10; i++) {</span>
+<span class="source-line-no">127</span><span id="line-127"> addEdits(log, hri, htd, 1, mvcc, scopes, "row");</span>
+<span class="source-line-no">128</span><span id="line-128"> }</span>
+<span class="source-line-no">129</span><span id="line-129"> } finally {</span>
+<span class="source-line-no">130</span><span id="line-130"> log.close();</span>
+<span class="source-line-no">131</span><span id="line-131"> }</span>
+<span class="source-line-no">132</span><span id="line-132"> }</span>
<span class="source-line-no">133</span><span id="line-133"></span>
-<span class="source-line-no">134</span><span id="line-134"> final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);</span>
-<span class="source-line-no">135</span><span id="line-135"> final CountDownLatch holdAppend = new CountDownLatch(1);</span>
-<span class="source-line-no">136</span><span id="line-136"> final CountDownLatch flushFinished = new CountDownLatch(1);</span>
-<span class="source-line-no">137</span><span id="line-137"> final CountDownLatch putFinished = new CountDownLatch(1);</span>
-<span class="source-line-no">138</span><span id="line-138"></span>
-<span class="source-line-no">139</span><span id="line-139"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
-<span class="source-line-no">140</span><span id="line-140"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
-<span class="source-line-no">141</span><span id="line-141"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
-<span class="source-line-no">142</span><span id="line-142"> log.init();</span>
-<span class="source-line-no">143</span><span id="line-143"> log.registerWALActionsListener(new WALActionsListener() {</span>
-<span class="source-line-no">144</span><span id="line-144"> @Override</span>
-<span class="source-line-no">145</span><span id="line-145"> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {</span>
-<span class="source-line-no">146</span><span id="line-146"> if (startHoldingForAppend.get()) {</span>
-<span class="source-line-no">147</span><span id="line-147"> try {</span>
-<span class="source-line-no">148</span><span id="line-148"> holdAppend.await();</span>
-<span class="source-line-no">149</span><span id="line-149"> } catch (InterruptedException e) {</span>
-<span class="source-line-no">150</span><span id="line-150"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">151</span><span id="line-151"> }</span>
-<span class="source-line-no">152</span><span id="line-152"> }</span>
-<span class="source-line-no">153</span><span id="line-153"> }</span>
-<span class="source-line-no">154</span><span id="line-154"> });</span>
-<span class="source-line-no">155</span><span id="line-155"></span>
-<span class="source-line-no">156</span><span id="line-156"> // open a new region which uses this WAL</span>
-<span class="source-line-no">157</span><span id="line-157"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
-<span class="source-line-no">158</span><span id="line-158"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();</span>
-<span class="source-line-no">159</span><span id="line-159"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
-<span class="source-line-no">160</span><span id="line-160"> ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,</span>
-<span class="source-line-no">161</span><span id="line-161"> MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);</span>
-<span class="source-line-no">162</span><span id="line-162"> final HRegion region = TEST_UTIL.createLocalHRegion(hri, CONF, htd, log);</span>
-<span class="source-line-no">163</span><span id="line-163"> ExecutorService exec = Executors.newFixedThreadPool(2);</span>
-<span class="source-line-no">164</span><span id="line-164"></span>
-<span class="source-line-no">165</span><span id="line-165"> // do a regular write first because of memstore size calculation.</span>
-<span class="source-line-no">166</span><span id="line-166"> region.put(new Put(b).addColumn(b, b, b));</span>
-<span class="source-line-no">167</span><span id="line-167"></span>
-<span class="source-line-no">168</span><span id="line-168"> startHoldingForAppend.set(true);</span>
-<span class="source-line-no">169</span><span id="line-169"> exec.submit(new Runnable() {</span>
-<span class="source-line-no">170</span><span id="line-170"> @Override</span>
-<span class="source-line-no">171</span><span id="line-171"> public void run() {</span>
-<span class="source-line-no">172</span><span id="line-172"> try {</span>
-<span class="source-line-no">173</span><span id="line-173"> region.put(new Put(b).addColumn(b, b, b));</span>
-<span class="source-line-no">174</span><span id="line-174"> putFinished.countDown();</span>
-<span class="source-line-no">175</span><span id="line-175"> } catch (IOException e) {</span>
-<span class="source-line-no">176</span><span id="line-176"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">177</span><span id="line-177"> }</span>
-<span class="source-line-no">178</span><span id="line-178"> }</span>
-<span class="source-line-no">179</span><span id="line-179"> });</span>
-<span class="source-line-no">180</span><span id="line-180"></span>
-<span class="source-line-no">181</span><span id="line-181"> // give the put a chance to start</span>
-<span class="source-line-no">182</span><span id="line-182"> Threads.sleep(3000);</span>
-<span class="source-line-no">183</span><span id="line-183"></span>
-<span class="source-line-no">184</span><span id="line-184"> exec.submit(new Runnable() {</span>
-<span class="source-line-no">185</span><span id="line-185"> @Override</span>
-<span class="source-line-no">186</span><span id="line-186"> public void run() {</span>
-<span class="source-line-no">187</span><span id="line-187"> try {</span>
-<span class="source-line-no">188</span><span id="line-188"> HRegion.FlushResult flushResult = region.flush(true);</span>
-<span class="source-line-no">189</span><span id="line-189"> LOG.info("Flush result:" + flushResult.getResult());</span>
-<span class="source-line-no">190</span><span id="line-190"> LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());</span>
-<span class="source-line-no">191</span><span id="line-191"> flushFinished.countDown();</span>
-<span class="source-line-no">192</span><span id="line-192"> } catch (IOException e) {</span>
-<span class="source-line-no">193</span><span id="line-193"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">194</span><span id="line-194"> }</span>
-<span class="source-line-no">195</span><span id="line-195"> }</span>
-<span class="source-line-no">196</span><span id="line-196"> });</span>
-<span class="source-line-no">197</span><span id="line-197"></span>
-<span class="source-line-no">198</span><span id="line-198"> // give the flush a chance to start. Flush should have got the region lock, and</span>
-<span class="source-line-no">199</span><span id="line-199"> // should have been waiting on the mvcc complete after this.</span>
-<span class="source-line-no">200</span><span id="line-200"> Threads.sleep(3000);</span>
-<span class="source-line-no">201</span><span id="line-201"></span>
-<span class="source-line-no">202</span><span id="line-202"> // let the append to WAL go through now that the flush already started</span>
-<span class="source-line-no">203</span><span id="line-203"> holdAppend.countDown();</span>
-<span class="source-line-no">204</span><span id="line-204"> putFinished.await();</span>
-<span class="source-line-no">205</span><span id="line-205"> flushFinished.await();</span>
-<span class="source-line-no">206</span><span id="line-206"></span>
-<span class="source-line-no">207</span><span id="line-207"> // check whether flush went through</span>
-<span class="source-line-no">208</span><span id="line-208"> assertEquals(1, region.getStoreFileList(new byte[][] { b }).size(), "Region did not flush?");</span>
-<span class="source-line-no">209</span><span id="line-209"></span>
-<span class="source-line-no">210</span><span id="line-210"> // now check the region's unflushed seqIds.</span>
-<span class="source-line-no">211</span><span id="line-211"> long seqId = AbstractTestFSWAL.getEarliestMemStoreSeqNum(log, hri.getEncodedNameAsBytes());</span>
-<span class="source-line-no">212</span><span id="line-212"> assertEquals(HConstants.NO_SEQNUM, seqId,</span>
-<span class="source-line-no">213</span><span id="line-213"> "Found seqId for the region which is already flushed");</span>
-<span class="source-line-no">214</span><span id="line-214"></span>
-<span class="source-line-no">215</span><span id="line-215"> region.close();</span>
-<span class="source-line-no">216</span><span id="line-216"> }</span>
-<span class="source-line-no">217</span><span id="line-217"> }</span>
-<span class="source-line-no">218</span><span id="line-218">}</span>
+<span class="source-line-no">134</span><span id="line-134"> /**</span>
+<span class="source-line-no">135</span><span id="line-135"> * Test case for https://issues.apache.org/jira/browse/HBASE-16721</span>
+<span class="source-line-no">136</span><span id="line-136"> */</span>
+<span class="source-line-no">137</span><span id="line-137"> @Test</span>
+<span class="source-line-no">138</span><span id="line-138"> public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {</span>
+<span class="source-line-no">139</span><span id="line-139"> final byte[] b = Bytes.toBytes("b");</span>
+<span class="source-line-no">140</span><span id="line-140"></span>
+<span class="source-line-no">141</span><span id="line-141"> final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);</span>
+<span class="source-line-no">142</span><span id="line-142"> final CountDownLatch holdAppend = new CountDownLatch(1);</span>
+<span class="source-line-no">143</span><span id="line-143"> final CountDownLatch flushFinished = new CountDownLatch(1);</span>
+<span class="source-line-no">144</span><span id="line-144"> final CountDownLatch putFinished = new CountDownLatch(1);</span>
+<span class="source-line-no">145</span><span id="line-145"></span>
+<span class="source-line-no">146</span><span id="line-146"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">147</span><span id="line-147"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">148</span><span id="line-148"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
+<span class="source-line-no">149</span><span id="line-149"> log.init();</span>
+<span class="source-line-no">150</span><span id="line-150"> log.registerWALActionsListener(new WALActionsListener() {</span>
+<span class="source-line-no">151</span><span id="line-151"> @Override</span>
+<span class="source-line-no">152</span><span id="line-152"> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {</span>
+<span class="source-line-no">153</span><span id="line-153"> if (startHoldingForAppend.get()) {</span>
+<span class="source-line-no">154</span><span id="line-154"> try {</span>
+<span class="source-line-no">155</span><span id="line-155"> holdAppend.await();</span>
+<span class="source-line-no">156</span><span id="line-156"> } catch (InterruptedException e) {</span>
+<span class="source-line-no">157</span><span id="line-157"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">158</span><span id="line-158"> }</span>
+<span class="source-line-no">159</span><span id="line-159"> }</span>
+<span class="source-line-no">160</span><span id="line-160"> }</span>
+<span class="source-line-no">161</span><span id="line-161"> });</span>
+<span class="source-line-no">162</span><span id="line-162"></span>
+<span class="source-line-no">163</span><span id="line-163"> // open a new region which uses this WAL</span>
+<span class="source-line-no">164</span><span id="line-164"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
+<span class="source-line-no">165</span><span id="line-165"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();</span>
+<span class="source-line-no">166</span><span id="line-166"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
+<span class="source-line-no">167</span><span id="line-167"> ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,</span>
+<span class="source-line-no">168</span><span id="line-168"> MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);</span>
+<span class="source-line-no">169</span><span id="line-169"> final HRegion region = TEST_UTIL.createLocalHRegion(hri, CONF, htd, log);</span>
+<span class="source-line-no">170</span><span id="line-170"> ExecutorService exec = Executors.newFixedThreadPool(2);</span>
+<span class="source-line-no">171</span><span id="line-171"></span>
+<span class="source-line-no">172</span><span id="line-172"> // do a regular write first because of memstore size calculation.</span>
+<span class="source-line-no">173</span><span id="line-173"> region.put(new Put(b).addColumn(b, b, b));</span>
+<span class="source-line-no">174</span><span id="line-174"></span>
+<span class="source-line-no">175</span><span id="line-175"> startHoldingForAppend.set(true);</span>
+<span class="source-line-no">176</span><span id="line-176"> exec.submit(new Runnable() {</span>
+<span class="source-line-no">177</span><span id="line-177"> @Override</span>
+<span class="source-line-no">178</span><span id="line-178"> public void run() {</span>
+<span class="source-line-no">179</span><span id="line-179"> try {</span>
+<span class="source-line-no">180</span><span id="line-180"> region.put(new Put(b).addColumn(b, b, b));</span>
+<span class="source-line-no">181</span><span id="line-181"> putFinished.countDown();</span>
+<span class="source-line-no">182</span><span id="line-182"> } catch (IOException e) {</span>
+<span class="source-line-no">183</span><span id="line-183"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">184</span><span id="line-184"> }</span>
+<span class="source-line-no">185</span><span id="line-185"> }</span>
+<span class="source-line-no">186</span><span id="line-186"> });</span>
+<span class="source-line-no">187</span><span id="line-187"></span>
+<span class="source-line-no">188</span><span id="line-188"> // give the put a chance to start</span>
+<span class="source-line-no">189</span><span id="line-189"> Threads.sleep(3000);</span>
+<span class="source-line-no">190</span><span id="line-190"></span>
+<span class="source-line-no">191</span><span id="line-191"> exec.submit(new Runnable() {</span>
+<span class="source-line-no">192</span><span id="line-192"> @Override</span>
+<span class="source-line-no">193</span><span id="line-193"> public void run() {</span>
+<span class="source-line-no">194</span><span id="line-194"> try {</span>
+<span class="source-line-no">195</span><span id="line-195"> HRegion.FlushResult flushResult = region.flush(true);</span>
+<span class="source-line-no">196</span><span id="line-196"> LOG.info("Flush result:" + flushResult.getResult());</span>
+<span class="source-line-no">197</span><span id="line-197"> LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());</span>
+<span class="source-line-no">198</span><span id="line-198"> flushFinished.countDown();</span>
+<span class="source-line-no">199</span><span id="line-199"> } catch (IOException e) {</span>
+<span class="source-line-no">200</span><span id="line-200"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">201</span><span id="line-201"> }</span>
+<span class="source-line-no">202</span><span id="line-202"> }</span>
+<span class="source-line-no">203</span><span id="line-203"> });</span>
+<span class="source-line-no">204</span><span id="line-204"></span>
+<span class="source-line-no">205</span><span id="line-205"> // give the flush a chance to start. Flush should have got the region lock, and</span>
+<span class="source-line-no">206</span><span id="line-206"> // should have been waiting on the mvcc complete after this.</span>
+<span class="source-line-no">207</span><span id="line-207"> Threads.sleep(3000);</span>
+<span class="source-line-no">208</span><span id="line-208"></span>
+<span class="source-line-no">209</span><span id="line-209"> // let the append to WAL go through now that the flush already started</span>
+<span class="source-line-no">210</span><span id="line-210"> holdAppend.countDown();</span>
+<span class="source-line-no">211</span><span id="line-211"> putFinished.await();</span>
+<span class="source-line-no">212</span><span id="line-212"> flushFinished.await();</span>
+<span class="source-line-no">213</span><span id="line-213"></span>
+<span class="source-line-no">214</span><span id="line-214"> // check whether flush went through</span>
+<span class="source-line-no">215</span><span id="line-215"> assertEquals(1, region.getStoreFileList(new byte[][] { b }).size(), "Region did not flush?");</span>
+<span class="source-line-no">216</span><span id="line-216"></span>
+<span class="source-line-no">217</span><span id="line-217"> // now check the region's unflushed seqIds.</span>
+<span class="source-line-no">218</span><span id="line-218"> long seqId = AbstractTestFSWAL.getEarliestMemStoreSeqNum(log, hri.getEncodedNameAsBytes());</span>
+<span class="source-line-no">219</span><span id="line-219"> assertEquals(HConstants.NO_SEQNUM, seqId,</span>
+<span class="source-line-no">220</span><span id="line-220"> "Found seqId for the region which is already flushed");</span>
+<span class="source-line-no">221</span><span id="line-221"></span>
+<span class="source-line-no">222</span><span id="line-222"> region.close();</span>
+<span class="source-line-no">223</span><span id="line-223"> }</span>
+<span class="source-line-no">224</span><span id="line-224"> }</span>
+<span class="source-line-no">225</span><span id="line-225"></span>
+<span class="source-line-no">226</span><span id="line-226"> /**</span>
+<span class="source-line-no">227</span><span id="line-227"> * Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the</span>
+<span class="source-line-no">228</span><span id="line-228"> * underlying DFSOutputStream#getPipeline() legitimately returns null (e.g. the DFS streamer is</span>
+<span class="source-line-no">229</span><span id="line-229"> * closed, or no block pipeline is currently established -- see HDFS-826 and the</span>
+<span class="source-line-no">230</span><span id="line-230"> * DFSOutputStream#getPipeline() javadoc: "returns the list of targets, if any"). Prior to this</span>
+<span class="source-line-no">231</span><span id="line-231"> * fix, AbstractFSWAL#rollWriterInternal's debug-log statement called</span>
+<span class="source-line-no">232</span><span id="line-232"> * Arrays.stream(getPipeline()), which threw a NullPointerException whenever this happened during</span>
+<span class="source-line-no">233</span><span id="line-233"> * a WAL roll, aborting the RegionServer on what was otherwise a successful roll.</span>
+<span class="source-line-no">234</span><span id="line-234"> */</span>
+<span class="source-line-no">235</span><span id="line-235"> @Test</span>
+<span class="source-line-no">236</span><span id="line-236"> public void testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone() throws Exception {</span>
+<span class="source-line-no">237</span><span id="line-237"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">238</span><span id="line-238"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">239</span><span id="line-239"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
+<span class="source-line-no">240</span><span id="line-240"> log.init();</span>
+<span class="source-line-no">241</span><span id="line-241"></span>
+<span class="source-line-no">242</span><span id="line-242"> // Simulate the legitimate HDFS contract: the wrapped DFSOutputStream currently has no</span>
+<span class="source-line-no">243</span><span id="line-243"> // established pipeline (e.g. streamer closed, or between blocks) and returns null.</span>
+<span class="source-line-no">244</span><span id="line-244"> DFSOutputStream mockDfsOut = mock(DFSOutputStream.class);</span>
+<span class="source-line-no">245</span><span id="line-245"> when(mockDfsOut.getPipeline()).thenReturn(null);</span>
+<span class="source-line-no">246</span><span id="line-246"> FSDataOutputStream wrappedOut = new HdfsDataOutputStream(mockDfsOut, null);</span>
+<span class="source-line-no">247</span><span id="line-247"></span>
+<span class="source-line-no">248</span><span id="line-248"> Field hdfsOutField = FSHLog.class.getDeclaredField("hdfs_out");</span>
+<span class="source-line-no">249</span><span id="line-249"> hdfsOutField.setAccessible(true);</span>
+<span class="source-line-no">250</span><span id="line-250"> hdfsOutField.set(log, wrappedOut);</span>
+<span class="source-line-no">251</span><span id="line-251"></span>
+<span class="source-line-no">252</span><span id="line-252"> DatanodeInfo[] pipeline = log.getPipeline();</span>
+<span class="source-line-no">253</span><span id="line-253"> assertNotNull(pipeline, "getPipeline() must never return null");</span>
+<span class="source-line-no">254</span><span id="line-254"> assertEquals(0, pipeline.length,</span>
+<span class="source-line-no">255</span><span id="line-255"> "Should normalize a null underlying pipeline to an empty array");</span>
+<span class="source-line-no">256</span><span id="line-256"> }</span>
+<span class="source-line-no">257</span><span id="line-257"> }</span>
+<span class="source-line-no">258</span><span id="line-258">}</span>
diff --git a/testdevapidocs/index-all.html b/testdevapidocs/index-all.html
index 1f4f4873c4e..02e32417160 100644
--- a/testdevapidocs/index-all.html
+++ b/testdevapidocs/index-all.html
@@ -68734,6 +68734,11 @@ loadScripts(document, 'script');</script>
<dd> </dd>
<dt><a href="org/apache/hadoop/hbase/client/TestConnectionUtils.html#testGetPauseTime()" class="member-name-link">testGetPauseTime()</a> - Method in class org.apache.hadoop.hbase.client.<a href="org/apache/hadoop/hbase/client/TestConnectionUtils.html" title="class in org.apache.hadoop.hbase.client">TestConnectionUtils</a></dt>
<dd> </dd>
+<dt><a href="org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()" class="member-name-link">testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()</a> - Method in class org.apache.hadoop.hbase.regionserver.wal.<a href="org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html" title="class in org.apache.hadoop.hbase.regionserver.wal">TestFSHLog</a></dt>
+<dd>
+<div class="block">Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the
+ underlying DFSOutputStream#getPipeline() legitimately returns null (e.g.</div>
+</dd>
<dt><a href="org/apache/hadoop/hbase/nio/TestMultiByteBuff.html#testGetPrimitivesWithSmallIndividualBBs()" class="member-name-link">testGetPrimitivesWithSmallIndividualBBs()</a> - Method in class org.apache.hadoop.hbase.nio.<a href="org/apache/hadoop/hbase/nio/TestMultiByteBuff.html" title="class in org.apache.hadoop.hbase.nio">TestMultiByteBuff</a></dt>
<dd> </dd>
<dt><a href="org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.html#testGetProcedure()" class="member-name-link">testGetProcedure()</a> - Method in class org.apache.hadoop.hbase.master.procedure.<a href="org/apache/hadoop/hbase/master/procedure/TestProcedureAdmin.html" title="class in org.apache.hadoop.hbase.master.procedure">TestProcedureAdmin</a></dt>
diff --git a/testdevapidocs/member-search-index.js b/testdevapidocs/member-search-index.js
index 2daf6a0dd71..c0cf2238a37 100644
--- a/testdevapidocs/member-search-index.js
+++ b/testdevapidocs/member-search-index.js
@@ -1 +1 @@
-memberSearchIndex = [{"p":"org.apache.hadoop.hbase.master.janitor","c":"TestMetaFixerNoCluster","l":"_ARI"},{"p":"org.apache.hadoop.hbase.io","c":"TestHalfStoreFileReader","l":"_b(String)","u":"_b(java.lang.String)"},{"p":"org.apache.hadoop.hbase.master.janitor","c":"TestMetaFixerNoCluster","l":"_BRI"},{"p":"org.apache.hadoop.hbase.backup","c":"TestSystemTableSnapshot","l":"_testBackupRestoreSystemTable()"},{"p":"org.apache.hadoop.hbase.regionserver","c":"TestBlocksScanned","l":"_testBlo [...]
\ No newline at end of file
+memberSearchIndex = [{"p":"org.apache.hadoop.hbase.master.janitor","c":"TestMetaFixerNoCluster","l":"_ARI"},{"p":"org.apache.hadoop.hbase.io","c":"TestHalfStoreFileReader","l":"_b(String)","u":"_b(java.lang.String)"},{"p":"org.apache.hadoop.hbase.master.janitor","c":"TestMetaFixerNoCluster","l":"_BRI"},{"p":"org.apache.hadoop.hbase.backup","c":"TestSystemTableSnapshot","l":"_testBackupRestoreSystemTable()"},{"p":"org.apache.hadoop.hbase.regionserver","c":"TestBlocksScanned","l":"_testBlo [...]
\ No newline at end of file
diff --git a/testdevapidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html b/testdevapidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
index 4da7ac5754e..a46aa49f689 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
@@ -80,7 +80,7 @@ loadScripts(document, 'script');</script>
<section class="class-description" id="class-description">
<hr>
<div class="type-signature"><span class="annotations">@Tag("org.apache.hadoop.hbase.testclassification.RegionServerTests") @Tag("org.apache.hadoop.hbase.testclassification.MediumTests")
-</span><span class="modifiers">public class </span><span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-61">TestFSHLog</a></span>
+</span><span class="modifiers">public class </span><span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-68">TestFSHLog</a></span>
<span class="extends-implements">extends <a href="AbstractTestFSWAL.html" title="class in org.apache.hadoop.hbase.regionserver.wal">AbstractTestFSWAL</a></span></div>
<div class="block">Provides FSHLog test cases.</div>
</section>
@@ -164,11 +164,17 @@ loadScripts(document, 'script');</script>
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a> suffix)</code></div>
<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"> </div>
<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
-<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#testSyncRunnerIndexOverflow()" class="member-name-link">testSyncRunnerIndexOverflow</a>()</code></div>
-<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"> </div>
+<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()" class="member-name-link">testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone</a>()</code></div>
+<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
+<div class="block">Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the
+ underlying DFSOutputStream#getPipeline() legitimately returns null (e.g.</div>
+</div>
<div class="col-first even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
-<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#testUnflushedSeqIdTracking()" class="member-name-link">testUnflushedSeqIdTracking</a>()</code></div>
-<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
+<div class="col-second even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#testSyncRunnerIndexOverflow()" class="member-name-link">testSyncRunnerIndexOverflow</a>()</code></div>
+<div class="col-last even-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"> </div>
+<div class="col-first odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code>void</code></div>
+<div class="col-second odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4"><code><a href="#testUnflushedSeqIdTracking()" class="member-name-link">testUnflushedSeqIdTracking</a>()</code></div>
+<div class="col-last odd-row-color method-summary-table method-summary-table-tab2 method-summary-table-tab4">
<div class="block">Test case for https://issues.apache.org/jira/browse/HBASE-16721</div>
</div>
</div>
@@ -194,7 +200,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="name">
<h3>name</h3>
-<div class="member-signature"><span class="modifiers">private</span> <span class="return-type"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-65">name</a></span></div>
+<div class="member-signature"><span class="modifiers">private</span> <span class="return-type"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-72">name</a></span></div>
</section>
</li>
</ul>
@@ -208,7 +214,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="<init>()">
<h3>TestFSHLog</h3>
-<div class="member-signature"><span class="modifiers">public</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-63">TestFSHLog</a></span>()</div>
+<div class="member-signature"><span class="modifiers">public</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-70">TestFSHLog</a></span>()</div>
</section>
</li>
</ul>
@@ -223,13 +229,13 @@ loadScripts(document, 'script');</script>
<section class="detail" id="initTestName(org.junit.jupiter.api.TestInfo)">
<h3>initTestName</h3>
<div class="member-signature"><span class="annotations">@BeforeEach
-</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-67">initTestName</a></span><wbr><span class="parameters">(org.junit.jupiter.api.TestInfo testInfo)</span></div>
+</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-74">initTestName</a></span><wbr><span class="parameters">(org.junit.jupiter.api.TestInfo testInfo)</span></div>
</section>
</li>
<li>
<section class="detail" id="newWAL(org.apache.hadoop.fs.FileSystem,org.apache.hadoop.fs.Path,java.lang.String,java.lang.String,org.apache.hadoop.conf.Configuration,java.util.List,boolean,java.lang.String,java.lang.String)">
<h3>newWAL</h3>
-<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type">org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-72">newWAL</a></span><wbr><span class="parameters">(org.apache.hadoop.fs.FileSystem fs,
+<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type">org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-79">newWAL</a></span><wbr><span class="parameters">(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootDir,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a> walDir,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a> archiveDir,
@@ -250,7 +256,7 @@ loadScripts(document, 'script');</script>
<li>
<section class="detail" id="newSlowWAL(org.apache.hadoop.fs.FileSystem,org.apache.hadoop.fs.Path,java.lang.String,java.lang.String,org.apache.hadoop.conf.Configuration,java.util.List,boolean,java.lang.String,java.lang.String,java.lang.Runnable)">
<h3>newSlowWAL</h3>
-<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type">org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-82">newSlowWAL</a></span><wbr><span class="parameters">(org.apache.hadoop.fs.FileSystem fs,
+<div class="member-signature"><span class="modifiers">protected</span> <span class="return-type">org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL<?></span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-89">newSlowWAL</a></span><wbr><span class="parameters">(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path rootDir,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a> walDir,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html" title="class or interface in java.lang" class="external-link">String</a> archiveDir,
@@ -273,7 +279,7 @@ loadScripts(document, 'script');</script>
<section class="detail" id="testSyncRunnerIndexOverflow()">
<h3>testSyncRunnerIndexOverflow</h3>
<div class="member-signature"><span class="annotations">@Test
-</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-100">testSyncRunnerIndexOverflow</a></span>()
+</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-107">testSyncRunnerIndexOverflow</a></span>()
throws <span class="exceptions"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/IOException.html" title="class or interface in java.io" class="external-link">IOException</a>,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/NoSuchFieldException.html" title="class or interface in java.lang" class="external-link">NoSuchFieldException</a>,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/SecurityException.html" title="class or interface in java.lang" class="external-link">SecurityException</a>,
@@ -293,7 +299,7 @@ loadScripts(document, 'script');</script>
<section class="detail" id="testUnflushedSeqIdTracking()">
<h3>testUnflushedSeqIdTracking</h3>
<div class="member-signature"><span class="annotations">@Test
-</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-130">testUnflushedSeqIdTracking</a></span>()
+</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-137">testUnflushedSeqIdTracking</a></span>()
throws <span class="exceptions"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/io/IOException.html" title="class or interface in java.io" class="external-link">IOException</a>,
<a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/InterruptedException.html" title="class or interface in java.lang" class="external-link">InterruptedException</a></span></div>
<div class="block">Test case for https://issues.apache.org/jira/browse/HBASE-16721</div>
@@ -304,6 +310,25 @@ loadScripts(document, 'script');</script>
</dl>
</section>
</li>
+<li>
+<section class="detail" id="testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone()">
+<h3>testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone</h3>
+<div class="member-signature"><span class="annotations">@Test
+</span><span class="modifiers">public</span> <span class="return-type">void</span> <span class="element-name"><a href="../../../../../../src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html#line-235">testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone</a></span>()
+ throws <span class="exceptions"><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Exception.html" title="class or interface in java.lang" class="external-link">Exception</a></span></div>
+<div class="block">Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the
+ underlying DFSOutputStream#getPipeline() legitimately returns null (e.g. the DFS streamer is
+ closed, or no block pipeline is currently established -- see HDFS-826 and the
+ DFSOutputStream#getPipeline() javadoc: "returns the list of targets, if any"). Prior to this
+ fix, AbstractFSWAL#rollWriterInternal's debug-log statement called
+ Arrays.stream(getPipeline()), which threw a NullPointerException whenever this happened during
+ a WAL roll, aborting the RegionServer on what was otherwise a successful roll.</div>
+<dl class="notes">
+<dt>Throws:</dt>
+<dd><code><a href="https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/Exception.html" title="class or interface in java.lang" class="external-link">Exception</a></code></dd>
+</dl>
+</section>
+</li>
</ul>
</section>
</li>
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
index 0704cea1e9c..d387103699d 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.html
@@ -31,204 +31,244 @@
<span class="source-line-no">018</span><span id="line-18">package org.apache.hadoop.hbase.regionserver.wal;</span>
<span class="source-line-no">019</span><span id="line-19"></span>
<span class="source-line-no">020</span><span id="line-20">import static org.junit.jupiter.api.Assertions.assertEquals;</span>
-<span class="source-line-no">021</span><span id="line-21"></span>
-<span class="source-line-no">022</span><span id="line-22">import java.io.IOException;</span>
-<span class="source-line-no">023</span><span id="line-23">import java.lang.reflect.Field;</span>
-<span class="source-line-no">024</span><span id="line-24">import java.util.List;</span>
-<span class="source-line-no">025</span><span id="line-25">import java.util.NavigableMap;</span>
-<span class="source-line-no">026</span><span id="line-26">import java.util.TreeMap;</span>
-<span class="source-line-no">027</span><span id="line-27">import java.util.concurrent.CountDownLatch;</span>
-<span class="source-line-no">028</span><span id="line-28">import java.util.concurrent.ExecutorService;</span>
-<span class="source-line-no">029</span><span id="line-29">import java.util.concurrent.Executors;</span>
-<span class="source-line-no">030</span><span id="line-30">import java.util.concurrent.atomic.AtomicBoolean;</span>
-<span class="source-line-no">031</span><span id="line-31">import org.apache.hadoop.conf.Configuration;</span>
-<span class="source-line-no">032</span><span id="line-32">import org.apache.hadoop.fs.FileSystem;</span>
-<span class="source-line-no">033</span><span id="line-33">import org.apache.hadoop.fs.Path;</span>
-<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.hbase.HConstants;</span>
-<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.hbase.TableName;</span>
-<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span>
-<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.hbase.client.Put;</span>
-<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.client.RegionInfo;</span>
-<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span>
-<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
-<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span>
-<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.regionserver.ChunkCreator;</span>
-<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.regionserver.HRegion;</span>
-<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.regionserver.MemStoreLAB;</span>
-<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;</span>
-<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.testclassification.MediumTests;</span>
-<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.testclassification.RegionServerTests;</span>
-<span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.util.Bytes;</span>
-<span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
-<span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.util.Threads;</span>
-<span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.wal.WALEdit;</span>
-<span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.wal.WALKey;</span>
-<span class="source-line-no">053</span><span id="line-53">import org.junit.jupiter.api.BeforeEach;</span>
-<span class="source-line-no">054</span><span id="line-54">import org.junit.jupiter.api.Tag;</span>
-<span class="source-line-no">055</span><span id="line-55">import org.junit.jupiter.api.Test;</span>
-<span class="source-line-no">056</span><span id="line-56">import org.junit.jupiter.api.TestInfo;</span>
-<span class="source-line-no">057</span><span id="line-57"></span>
-<span class="source-line-no">058</span><span id="line-58">/**</span>
-<span class="source-line-no">059</span><span id="line-59"> * Provides FSHLog test cases.</span>
-<span class="source-line-no">060</span><span id="line-60"> */</span>
-<span class="source-line-no">061</span><span id="line-61">@Tag(RegionServerTests.TAG)</span>
-<span class="source-line-no">062</span><span id="line-62">@Tag(MediumTests.TAG)</span>
-<span class="source-line-no">063</span><span id="line-63">public class TestFSHLog extends AbstractTestFSWAL {</span>
+<span class="source-line-no">021</span><span id="line-21">import static org.junit.jupiter.api.Assertions.assertNotNull;</span>
+<span class="source-line-no">022</span><span id="line-22">import static org.mockito.Mockito.mock;</span>
+<span class="source-line-no">023</span><span id="line-23">import static org.mockito.Mockito.when;</span>
+<span class="source-line-no">024</span><span id="line-24"></span>
+<span class="source-line-no">025</span><span id="line-25">import java.io.IOException;</span>
+<span class="source-line-no">026</span><span id="line-26">import java.lang.reflect.Field;</span>
+<span class="source-line-no">027</span><span id="line-27">import java.util.List;</span>
+<span class="source-line-no">028</span><span id="line-28">import java.util.NavigableMap;</span>
+<span class="source-line-no">029</span><span id="line-29">import java.util.TreeMap;</span>
+<span class="source-line-no">030</span><span id="line-30">import java.util.concurrent.CountDownLatch;</span>
+<span class="source-line-no">031</span><span id="line-31">import java.util.concurrent.ExecutorService;</span>
+<span class="source-line-no">032</span><span id="line-32">import java.util.concurrent.Executors;</span>
+<span class="source-line-no">033</span><span id="line-33">import java.util.concurrent.atomic.AtomicBoolean;</span>
+<span class="source-line-no">034</span><span id="line-34">import org.apache.hadoop.conf.Configuration;</span>
+<span class="source-line-no">035</span><span id="line-35">import org.apache.hadoop.fs.FSDataOutputStream;</span>
+<span class="source-line-no">036</span><span id="line-36">import org.apache.hadoop.fs.FileSystem;</span>
+<span class="source-line-no">037</span><span id="line-37">import org.apache.hadoop.fs.Path;</span>
+<span class="source-line-no">038</span><span id="line-38">import org.apache.hadoop.hbase.HConstants;</span>
+<span class="source-line-no">039</span><span id="line-39">import org.apache.hadoop.hbase.TableName;</span>
+<span class="source-line-no">040</span><span id="line-40">import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;</span>
+<span class="source-line-no">041</span><span id="line-41">import org.apache.hadoop.hbase.client.Put;</span>
+<span class="source-line-no">042</span><span id="line-42">import org.apache.hadoop.hbase.client.RegionInfo;</span>
+<span class="source-line-no">043</span><span id="line-43">import org.apache.hadoop.hbase.client.RegionInfoBuilder;</span>
+<span class="source-line-no">044</span><span id="line-44">import org.apache.hadoop.hbase.client.TableDescriptor;</span>
+<span class="source-line-no">045</span><span id="line-45">import org.apache.hadoop.hbase.client.TableDescriptorBuilder;</span>
+<span class="source-line-no">046</span><span id="line-46">import org.apache.hadoop.hbase.regionserver.ChunkCreator;</span>
+<span class="source-line-no">047</span><span id="line-47">import org.apache.hadoop.hbase.regionserver.HRegion;</span>
+<span class="source-line-no">048</span><span id="line-48">import org.apache.hadoop.hbase.regionserver.MemStoreLAB;</span>
+<span class="source-line-no">049</span><span id="line-49">import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;</span>
+<span class="source-line-no">050</span><span id="line-50">import org.apache.hadoop.hbase.testclassification.MediumTests;</span>
+<span class="source-line-no">051</span><span id="line-51">import org.apache.hadoop.hbase.testclassification.RegionServerTests;</span>
+<span class="source-line-no">052</span><span id="line-52">import org.apache.hadoop.hbase.util.Bytes;</span>
+<span class="source-line-no">053</span><span id="line-53">import org.apache.hadoop.hbase.util.CommonFSUtils;</span>
+<span class="source-line-no">054</span><span id="line-54">import org.apache.hadoop.hbase.util.Threads;</span>
+<span class="source-line-no">055</span><span id="line-55">import org.apache.hadoop.hbase.wal.WALEdit;</span>
+<span class="source-line-no">056</span><span id="line-56">import org.apache.hadoop.hbase.wal.WALKey;</span>
+<span class="source-line-no">057</span><span id="line-57">import org.apache.hadoop.hdfs.DFSOutputStream;</span>
+<span class="source-line-no">058</span><span id="line-58">import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;</span>
+<span class="source-line-no">059</span><span id="line-59">import org.apache.hadoop.hdfs.protocol.DatanodeInfo;</span>
+<span class="source-line-no">060</span><span id="line-60">import org.junit.jupiter.api.BeforeEach;</span>
+<span class="source-line-no">061</span><span id="line-61">import org.junit.jupiter.api.Tag;</span>
+<span class="source-line-no">062</span><span id="line-62">import org.junit.jupiter.api.Test;</span>
+<span class="source-line-no">063</span><span id="line-63">import org.junit.jupiter.api.TestInfo;</span>
<span class="source-line-no">064</span><span id="line-64"></span>
-<span class="source-line-no">065</span><span id="line-65"> private String name;</span>
-<span class="source-line-no">066</span><span id="line-66"></span>
-<span class="source-line-no">067</span><span id="line-67"> @BeforeEach</span>
-<span class="source-line-no">068</span><span id="line-68"> public void initTestName(TestInfo testInfo) {</span>
-<span class="source-line-no">069</span><span id="line-69"> name = testInfo.getTestMethod().get().getName();</span>
-<span class="source-line-no">070</span><span id="line-70"> }</span>
+<span class="source-line-no">065</span><span id="line-65">/**</span>
+<span class="source-line-no">066</span><span id="line-66"> * Provides FSHLog test cases.</span>
+<span class="source-line-no">067</span><span id="line-67"> */</span>
+<span class="source-line-no">068</span><span id="line-68">@Tag(RegionServerTests.TAG)</span>
+<span class="source-line-no">069</span><span id="line-69">@Tag(MediumTests.TAG)</span>
+<span class="source-line-no">070</span><span id="line-70">public class TestFSHLog extends AbstractTestFSWAL {</span>
<span class="source-line-no">071</span><span id="line-71"></span>
-<span class="source-line-no">072</span><span id="line-72"> @Override</span>
-<span class="source-line-no">073</span><span id="line-73"> protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String walDir, String archiveDir,</span>
-<span class="source-line-no">074</span><span id="line-74"> Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix,</span>
-<span class="source-line-no">075</span><span id="line-75"> String suffix) throws IOException {</span>
-<span class="source-line-no">076</span><span id="line-76"> FSHLog wal =</span>
-<span class="source-line-no">077</span><span id="line-77"> new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);</span>
-<span class="source-line-no">078</span><span id="line-78"> wal.init();</span>
-<span class="source-line-no">079</span><span id="line-79"> return wal;</span>
-<span class="source-line-no">080</span><span id="line-80"> }</span>
-<span class="source-line-no">081</span><span id="line-81"></span>
-<span class="source-line-no">082</span><span id="line-82"> @Override</span>
-<span class="source-line-no">083</span><span id="line-83"> protected AbstractFSWAL<?> newSlowWAL(FileSystem fs, Path rootDir, String walDir,</span>
-<span class="source-line-no">084</span><span id="line-84"> String archiveDir, Configuration conf, List<WALActionsListener> listeners,</span>
-<span class="source-line-no">085</span><span id="line-85"> boolean failIfWALExists, String prefix, String suffix, final Runnable action)</span>
-<span class="source-line-no">086</span><span id="line-86"> throws IOException {</span>
-<span class="source-line-no">087</span><span id="line-87"> FSHLog wal = new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists,</span>
-<span class="source-line-no">088</span><span id="line-88"> prefix, suffix) {</span>
-<span class="source-line-no">089</span><span id="line-89"></span>
-<span class="source-line-no">090</span><span id="line-90"> @Override</span>
-<span class="source-line-no">091</span><span id="line-91"> protected void atHeadOfRingBufferEventHandlerAppend() {</span>
-<span class="source-line-no">092</span><span id="line-92"> action.run();</span>
-<span class="source-line-no">093</span><span id="line-93"> super.atHeadOfRingBufferEventHandlerAppend();</span>
-<span class="source-line-no">094</span><span id="line-94"> }</span>
-<span class="source-line-no">095</span><span id="line-95"> };</span>
-<span class="source-line-no">096</span><span id="line-96"> wal.init();</span>
-<span class="source-line-no">097</span><span id="line-97"> return wal;</span>
-<span class="source-line-no">098</span><span id="line-98"> }</span>
-<span class="source-line-no">099</span><span id="line-99"></span>
-<span class="source-line-no">100</span><span id="line-100"> @Test</span>
-<span class="source-line-no">101</span><span id="line-101"> public void testSyncRunnerIndexOverflow() throws IOException, NoSuchFieldException,</span>
-<span class="source-line-no">102</span><span id="line-102"> SecurityException, IllegalArgumentException, IllegalAccessException {</span>
-<span class="source-line-no">103</span><span id="line-103"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
-<span class="source-line-no">104</span><span id="line-104"> FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
-<span class="source-line-no">105</span><span id="line-105"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);</span>
-<span class="source-line-no">106</span><span id="line-106"> log.init();</span>
-<span class="source-line-no">107</span><span id="line-107"> try {</span>
-<span class="source-line-no">108</span><span id="line-108"> Field syncRunnerIndexField = FSHLog.class.getDeclaredField("syncRunnerIndex");</span>
-<span class="source-line-no">109</span><span id="line-109"> syncRunnerIndexField.setAccessible(true);</span>
-<span class="source-line-no">110</span><span id="line-110"> syncRunnerIndexField.set(log, Integer.MAX_VALUE - 1);</span>
-<span class="source-line-no">111</span><span id="line-111"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
-<span class="source-line-no">112</span><span id="line-112"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();</span>
-<span class="source-line-no">113</span><span id="line-113"> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span>
-<span class="source-line-no">114</span><span id="line-114"> for (byte[] fam : htd.getColumnFamilyNames()) {</span>
-<span class="source-line-no">115</span><span id="line-115"> scopes.put(fam, 0);</span>
-<span class="source-line-no">116</span><span id="line-116"> }</span>
-<span class="source-line-no">117</span><span id="line-117"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
-<span class="source-line-no">118</span><span id="line-118"> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();</span>
-<span class="source-line-no">119</span><span id="line-119"> for (int i = 0; i < 10; i++) {</span>
-<span class="source-line-no">120</span><span id="line-120"> addEdits(log, hri, htd, 1, mvcc, scopes, "row");</span>
-<span class="source-line-no">121</span><span id="line-121"> }</span>
-<span class="source-line-no">122</span><span id="line-122"> } finally {</span>
-<span class="source-line-no">123</span><span id="line-123"> log.close();</span>
-<span class="source-line-no">124</span><span id="line-124"> }</span>
-<span class="source-line-no">125</span><span id="line-125"> }</span>
-<span class="source-line-no">126</span><span id="line-126"></span>
-<span class="source-line-no">127</span><span id="line-127"> /**</span>
-<span class="source-line-no">128</span><span id="line-128"> * Test case for https://issues.apache.org/jira/browse/HBASE-16721</span>
-<span class="source-line-no">129</span><span id="line-129"> */</span>
-<span class="source-line-no">130</span><span id="line-130"> @Test</span>
-<span class="source-line-no">131</span><span id="line-131"> public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {</span>
-<span class="source-line-no">132</span><span id="line-132"> final byte[] b = Bytes.toBytes("b");</span>
+<span class="source-line-no">072</span><span id="line-72"> private String name;</span>
+<span class="source-line-no">073</span><span id="line-73"></span>
+<span class="source-line-no">074</span><span id="line-74"> @BeforeEach</span>
+<span class="source-line-no">075</span><span id="line-75"> public void initTestName(TestInfo testInfo) {</span>
+<span class="source-line-no">076</span><span id="line-76"> name = testInfo.getTestMethod().get().getName();</span>
+<span class="source-line-no">077</span><span id="line-77"> }</span>
+<span class="source-line-no">078</span><span id="line-78"></span>
+<span class="source-line-no">079</span><span id="line-79"> @Override</span>
+<span class="source-line-no">080</span><span id="line-80"> protected AbstractFSWAL<?> newWAL(FileSystem fs, Path rootDir, String walDir, String archiveDir,</span>
+<span class="source-line-no">081</span><span id="line-81"> Configuration conf, List<WALActionsListener> listeners, boolean failIfWALExists, String prefix,</span>
+<span class="source-line-no">082</span><span id="line-82"> String suffix) throws IOException {</span>
+<span class="source-line-no">083</span><span id="line-83"> FSHLog wal =</span>
+<span class="source-line-no">084</span><span id="line-84"> new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);</span>
+<span class="source-line-no">085</span><span id="line-85"> wal.init();</span>
+<span class="source-line-no">086</span><span id="line-86"> return wal;</span>
+<span class="source-line-no">087</span><span id="line-87"> }</span>
+<span class="source-line-no">088</span><span id="line-88"></span>
+<span class="source-line-no">089</span><span id="line-89"> @Override</span>
+<span class="source-line-no">090</span><span id="line-90"> protected AbstractFSWAL<?> newSlowWAL(FileSystem fs, Path rootDir, String walDir,</span>
+<span class="source-line-no">091</span><span id="line-91"> String archiveDir, Configuration conf, List<WALActionsListener> listeners,</span>
+<span class="source-line-no">092</span><span id="line-92"> boolean failIfWALExists, String prefix, String suffix, final Runnable action)</span>
+<span class="source-line-no">093</span><span id="line-93"> throws IOException {</span>
+<span class="source-line-no">094</span><span id="line-94"> FSHLog wal = new FSHLog(fs, rootDir, walDir, archiveDir, conf, listeners, failIfWALExists,</span>
+<span class="source-line-no">095</span><span id="line-95"> prefix, suffix) {</span>
+<span class="source-line-no">096</span><span id="line-96"></span>
+<span class="source-line-no">097</span><span id="line-97"> @Override</span>
+<span class="source-line-no">098</span><span id="line-98"> protected void atHeadOfRingBufferEventHandlerAppend() {</span>
+<span class="source-line-no">099</span><span id="line-99"> action.run();</span>
+<span class="source-line-no">100</span><span id="line-100"> super.atHeadOfRingBufferEventHandlerAppend();</span>
+<span class="source-line-no">101</span><span id="line-101"> }</span>
+<span class="source-line-no">102</span><span id="line-102"> };</span>
+<span class="source-line-no">103</span><span id="line-103"> wal.init();</span>
+<span class="source-line-no">104</span><span id="line-104"> return wal;</span>
+<span class="source-line-no">105</span><span id="line-105"> }</span>
+<span class="source-line-no">106</span><span id="line-106"></span>
+<span class="source-line-no">107</span><span id="line-107"> @Test</span>
+<span class="source-line-no">108</span><span id="line-108"> public void testSyncRunnerIndexOverflow() throws IOException, NoSuchFieldException,</span>
+<span class="source-line-no">109</span><span id="line-109"> SecurityException, IllegalArgumentException, IllegalAccessException {</span>
+<span class="source-line-no">110</span><span id="line-110"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">111</span><span id="line-111"> FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">112</span><span id="line-112"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);</span>
+<span class="source-line-no">113</span><span id="line-113"> log.init();</span>
+<span class="source-line-no">114</span><span id="line-114"> try {</span>
+<span class="source-line-no">115</span><span id="line-115"> Field syncRunnerIndexField = FSHLog.class.getDeclaredField("syncRunnerIndex");</span>
+<span class="source-line-no">116</span><span id="line-116"> syncRunnerIndexField.setAccessible(true);</span>
+<span class="source-line-no">117</span><span id="line-117"> syncRunnerIndexField.set(log, Integer.MAX_VALUE - 1);</span>
+<span class="source-line-no">118</span><span id="line-118"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
+<span class="source-line-no">119</span><span id="line-119"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();</span>
+<span class="source-line-no">120</span><span id="line-120"> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);</span>
+<span class="source-line-no">121</span><span id="line-121"> for (byte[] fam : htd.getColumnFamilyNames()) {</span>
+<span class="source-line-no">122</span><span id="line-122"> scopes.put(fam, 0);</span>
+<span class="source-line-no">123</span><span id="line-123"> }</span>
+<span class="source-line-no">124</span><span id="line-124"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
+<span class="source-line-no">125</span><span id="line-125"> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();</span>
+<span class="source-line-no">126</span><span id="line-126"> for (int i = 0; i < 10; i++) {</span>
+<span class="source-line-no">127</span><span id="line-127"> addEdits(log, hri, htd, 1, mvcc, scopes, "row");</span>
+<span class="source-line-no">128</span><span id="line-128"> }</span>
+<span class="source-line-no">129</span><span id="line-129"> } finally {</span>
+<span class="source-line-no">130</span><span id="line-130"> log.close();</span>
+<span class="source-line-no">131</span><span id="line-131"> }</span>
+<span class="source-line-no">132</span><span id="line-132"> }</span>
<span class="source-line-no">133</span><span id="line-133"></span>
-<span class="source-line-no">134</span><span id="line-134"> final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);</span>
-<span class="source-line-no">135</span><span id="line-135"> final CountDownLatch holdAppend = new CountDownLatch(1);</span>
-<span class="source-line-no">136</span><span id="line-136"> final CountDownLatch flushFinished = new CountDownLatch(1);</span>
-<span class="source-line-no">137</span><span id="line-137"> final CountDownLatch putFinished = new CountDownLatch(1);</span>
-<span class="source-line-no">138</span><span id="line-138"></span>
-<span class="source-line-no">139</span><span id="line-139"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
-<span class="source-line-no">140</span><span id="line-140"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
-<span class="source-line-no">141</span><span id="line-141"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
-<span class="source-line-no">142</span><span id="line-142"> log.init();</span>
-<span class="source-line-no">143</span><span id="line-143"> log.registerWALActionsListener(new WALActionsListener() {</span>
-<span class="source-line-no">144</span><span id="line-144"> @Override</span>
-<span class="source-line-no">145</span><span id="line-145"> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {</span>
-<span class="source-line-no">146</span><span id="line-146"> if (startHoldingForAppend.get()) {</span>
-<span class="source-line-no">147</span><span id="line-147"> try {</span>
-<span class="source-line-no">148</span><span id="line-148"> holdAppend.await();</span>
-<span class="source-line-no">149</span><span id="line-149"> } catch (InterruptedException e) {</span>
-<span class="source-line-no">150</span><span id="line-150"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">151</span><span id="line-151"> }</span>
-<span class="source-line-no">152</span><span id="line-152"> }</span>
-<span class="source-line-no">153</span><span id="line-153"> }</span>
-<span class="source-line-no">154</span><span id="line-154"> });</span>
-<span class="source-line-no">155</span><span id="line-155"></span>
-<span class="source-line-no">156</span><span id="line-156"> // open a new region which uses this WAL</span>
-<span class="source-line-no">157</span><span id="line-157"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
-<span class="source-line-no">158</span><span id="line-158"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();</span>
-<span class="source-line-no">159</span><span id="line-159"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
-<span class="source-line-no">160</span><span id="line-160"> ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,</span>
-<span class="source-line-no">161</span><span id="line-161"> MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);</span>
-<span class="source-line-no">162</span><span id="line-162"> final HRegion region = TEST_UTIL.createLocalHRegion(hri, CONF, htd, log);</span>
-<span class="source-line-no">163</span><span id="line-163"> ExecutorService exec = Executors.newFixedThreadPool(2);</span>
-<span class="source-line-no">164</span><span id="line-164"></span>
-<span class="source-line-no">165</span><span id="line-165"> // do a regular write first because of memstore size calculation.</span>
-<span class="source-line-no">166</span><span id="line-166"> region.put(new Put(b).addColumn(b, b, b));</span>
-<span class="source-line-no">167</span><span id="line-167"></span>
-<span class="source-line-no">168</span><span id="line-168"> startHoldingForAppend.set(true);</span>
-<span class="source-line-no">169</span><span id="line-169"> exec.submit(new Runnable() {</span>
-<span class="source-line-no">170</span><span id="line-170"> @Override</span>
-<span class="source-line-no">171</span><span id="line-171"> public void run() {</span>
-<span class="source-line-no">172</span><span id="line-172"> try {</span>
-<span class="source-line-no">173</span><span id="line-173"> region.put(new Put(b).addColumn(b, b, b));</span>
-<span class="source-line-no">174</span><span id="line-174"> putFinished.countDown();</span>
-<span class="source-line-no">175</span><span id="line-175"> } catch (IOException e) {</span>
-<span class="source-line-no">176</span><span id="line-176"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">177</span><span id="line-177"> }</span>
-<span class="source-line-no">178</span><span id="line-178"> }</span>
-<span class="source-line-no">179</span><span id="line-179"> });</span>
-<span class="source-line-no">180</span><span id="line-180"></span>
-<span class="source-line-no">181</span><span id="line-181"> // give the put a chance to start</span>
-<span class="source-line-no">182</span><span id="line-182"> Threads.sleep(3000);</span>
-<span class="source-line-no">183</span><span id="line-183"></span>
-<span class="source-line-no">184</span><span id="line-184"> exec.submit(new Runnable() {</span>
-<span class="source-line-no">185</span><span id="line-185"> @Override</span>
-<span class="source-line-no">186</span><span id="line-186"> public void run() {</span>
-<span class="source-line-no">187</span><span id="line-187"> try {</span>
-<span class="source-line-no">188</span><span id="line-188"> HRegion.FlushResult flushResult = region.flush(true);</span>
-<span class="source-line-no">189</span><span id="line-189"> LOG.info("Flush result:" + flushResult.getResult());</span>
-<span class="source-line-no">190</span><span id="line-190"> LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());</span>
-<span class="source-line-no">191</span><span id="line-191"> flushFinished.countDown();</span>
-<span class="source-line-no">192</span><span id="line-192"> } catch (IOException e) {</span>
-<span class="source-line-no">193</span><span id="line-193"> LOG.error(e.toString(), e);</span>
-<span class="source-line-no">194</span><span id="line-194"> }</span>
-<span class="source-line-no">195</span><span id="line-195"> }</span>
-<span class="source-line-no">196</span><span id="line-196"> });</span>
-<span class="source-line-no">197</span><span id="line-197"></span>
-<span class="source-line-no">198</span><span id="line-198"> // give the flush a chance to start. Flush should have got the region lock, and</span>
-<span class="source-line-no">199</span><span id="line-199"> // should have been waiting on the mvcc complete after this.</span>
-<span class="source-line-no">200</span><span id="line-200"> Threads.sleep(3000);</span>
-<span class="source-line-no">201</span><span id="line-201"></span>
-<span class="source-line-no">202</span><span id="line-202"> // let the append to WAL go through now that the flush already started</span>
-<span class="source-line-no">203</span><span id="line-203"> holdAppend.countDown();</span>
-<span class="source-line-no">204</span><span id="line-204"> putFinished.await();</span>
-<span class="source-line-no">205</span><span id="line-205"> flushFinished.await();</span>
-<span class="source-line-no">206</span><span id="line-206"></span>
-<span class="source-line-no">207</span><span id="line-207"> // check whether flush went through</span>
-<span class="source-line-no">208</span><span id="line-208"> assertEquals(1, region.getStoreFileList(new byte[][] { b }).size(), "Region did not flush?");</span>
-<span class="source-line-no">209</span><span id="line-209"></span>
-<span class="source-line-no">210</span><span id="line-210"> // now check the region's unflushed seqIds.</span>
-<span class="source-line-no">211</span><span id="line-211"> long seqId = AbstractTestFSWAL.getEarliestMemStoreSeqNum(log, hri.getEncodedNameAsBytes());</span>
-<span class="source-line-no">212</span><span id="line-212"> assertEquals(HConstants.NO_SEQNUM, seqId,</span>
-<span class="source-line-no">213</span><span id="line-213"> "Found seqId for the region which is already flushed");</span>
-<span class="source-line-no">214</span><span id="line-214"></span>
-<span class="source-line-no">215</span><span id="line-215"> region.close();</span>
-<span class="source-line-no">216</span><span id="line-216"> }</span>
-<span class="source-line-no">217</span><span id="line-217"> }</span>
-<span class="source-line-no">218</span><span id="line-218">}</span>
+<span class="source-line-no">134</span><span id="line-134"> /**</span>
+<span class="source-line-no">135</span><span id="line-135"> * Test case for https://issues.apache.org/jira/browse/HBASE-16721</span>
+<span class="source-line-no">136</span><span id="line-136"> */</span>
+<span class="source-line-no">137</span><span id="line-137"> @Test</span>
+<span class="source-line-no">138</span><span id="line-138"> public void testUnflushedSeqIdTracking() throws IOException, InterruptedException {</span>
+<span class="source-line-no">139</span><span id="line-139"> final byte[] b = Bytes.toBytes("b");</span>
+<span class="source-line-no">140</span><span id="line-140"></span>
+<span class="source-line-no">141</span><span id="line-141"> final AtomicBoolean startHoldingForAppend = new AtomicBoolean(false);</span>
+<span class="source-line-no">142</span><span id="line-142"> final CountDownLatch holdAppend = new CountDownLatch(1);</span>
+<span class="source-line-no">143</span><span id="line-143"> final CountDownLatch flushFinished = new CountDownLatch(1);</span>
+<span class="source-line-no">144</span><span id="line-144"> final CountDownLatch putFinished = new CountDownLatch(1);</span>
+<span class="source-line-no">145</span><span id="line-145"></span>
+<span class="source-line-no">146</span><span id="line-146"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">147</span><span id="line-147"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">148</span><span id="line-148"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
+<span class="source-line-no">149</span><span id="line-149"> log.init();</span>
+<span class="source-line-no">150</span><span id="line-150"> log.registerWALActionsListener(new WALActionsListener() {</span>
+<span class="source-line-no">151</span><span id="line-151"> @Override</span>
+<span class="source-line-no">152</span><span id="line-152"> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {</span>
+<span class="source-line-no">153</span><span id="line-153"> if (startHoldingForAppend.get()) {</span>
+<span class="source-line-no">154</span><span id="line-154"> try {</span>
+<span class="source-line-no">155</span><span id="line-155"> holdAppend.await();</span>
+<span class="source-line-no">156</span><span id="line-156"> } catch (InterruptedException e) {</span>
+<span class="source-line-no">157</span><span id="line-157"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">158</span><span id="line-158"> }</span>
+<span class="source-line-no">159</span><span id="line-159"> }</span>
+<span class="source-line-no">160</span><span id="line-160"> }</span>
+<span class="source-line-no">161</span><span id="line-161"> });</span>
+<span class="source-line-no">162</span><span id="line-162"></span>
+<span class="source-line-no">163</span><span id="line-163"> // open a new region which uses this WAL</span>
+<span class="source-line-no">164</span><span id="line-164"> TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name))</span>
+<span class="source-line-no">165</span><span id="line-165"> .setColumnFamily(ColumnFamilyDescriptorBuilder.of(b)).build();</span>
+<span class="source-line-no">166</span><span id="line-166"> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();</span>
+<span class="source-line-no">167</span><span id="line-167"> ChunkCreator.initialize(MemStoreLAB.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null,</span>
+<span class="source-line-no">168</span><span id="line-168"> MemStoreLAB.INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT);</span>
+<span class="source-line-no">169</span><span id="line-169"> final HRegion region = TEST_UTIL.createLocalHRegion(hri, CONF, htd, log);</span>
+<span class="source-line-no">170</span><span id="line-170"> ExecutorService exec = Executors.newFixedThreadPool(2);</span>
+<span class="source-line-no">171</span><span id="line-171"></span>
+<span class="source-line-no">172</span><span id="line-172"> // do a regular write first because of memstore size calculation.</span>
+<span class="source-line-no">173</span><span id="line-173"> region.put(new Put(b).addColumn(b, b, b));</span>
+<span class="source-line-no">174</span><span id="line-174"></span>
+<span class="source-line-no">175</span><span id="line-175"> startHoldingForAppend.set(true);</span>
+<span class="source-line-no">176</span><span id="line-176"> exec.submit(new Runnable() {</span>
+<span class="source-line-no">177</span><span id="line-177"> @Override</span>
+<span class="source-line-no">178</span><span id="line-178"> public void run() {</span>
+<span class="source-line-no">179</span><span id="line-179"> try {</span>
+<span class="source-line-no">180</span><span id="line-180"> region.put(new Put(b).addColumn(b, b, b));</span>
+<span class="source-line-no">181</span><span id="line-181"> putFinished.countDown();</span>
+<span class="source-line-no">182</span><span id="line-182"> } catch (IOException e) {</span>
+<span class="source-line-no">183</span><span id="line-183"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">184</span><span id="line-184"> }</span>
+<span class="source-line-no">185</span><span id="line-185"> }</span>
+<span class="source-line-no">186</span><span id="line-186"> });</span>
+<span class="source-line-no">187</span><span id="line-187"></span>
+<span class="source-line-no">188</span><span id="line-188"> // give the put a chance to start</span>
+<span class="source-line-no">189</span><span id="line-189"> Threads.sleep(3000);</span>
+<span class="source-line-no">190</span><span id="line-190"></span>
+<span class="source-line-no">191</span><span id="line-191"> exec.submit(new Runnable() {</span>
+<span class="source-line-no">192</span><span id="line-192"> @Override</span>
+<span class="source-line-no">193</span><span id="line-193"> public void run() {</span>
+<span class="source-line-no">194</span><span id="line-194"> try {</span>
+<span class="source-line-no">195</span><span id="line-195"> HRegion.FlushResult flushResult = region.flush(true);</span>
+<span class="source-line-no">196</span><span id="line-196"> LOG.info("Flush result:" + flushResult.getResult());</span>
+<span class="source-line-no">197</span><span id="line-197"> LOG.info("Flush succeeded:" + flushResult.isFlushSucceeded());</span>
+<span class="source-line-no">198</span><span id="line-198"> flushFinished.countDown();</span>
+<span class="source-line-no">199</span><span id="line-199"> } catch (IOException e) {</span>
+<span class="source-line-no">200</span><span id="line-200"> LOG.error(e.toString(), e);</span>
+<span class="source-line-no">201</span><span id="line-201"> }</span>
+<span class="source-line-no">202</span><span id="line-202"> }</span>
+<span class="source-line-no">203</span><span id="line-203"> });</span>
+<span class="source-line-no">204</span><span id="line-204"></span>
+<span class="source-line-no">205</span><span id="line-205"> // give the flush a chance to start. Flush should have got the region lock, and</span>
+<span class="source-line-no">206</span><span id="line-206"> // should have been waiting on the mvcc complete after this.</span>
+<span class="source-line-no">207</span><span id="line-207"> Threads.sleep(3000);</span>
+<span class="source-line-no">208</span><span id="line-208"></span>
+<span class="source-line-no">209</span><span id="line-209"> // let the append to WAL go through now that the flush already started</span>
+<span class="source-line-no">210</span><span id="line-210"> holdAppend.countDown();</span>
+<span class="source-line-no">211</span><span id="line-211"> putFinished.await();</span>
+<span class="source-line-no">212</span><span id="line-212"> flushFinished.await();</span>
+<span class="source-line-no">213</span><span id="line-213"></span>
+<span class="source-line-no">214</span><span id="line-214"> // check whether flush went through</span>
+<span class="source-line-no">215</span><span id="line-215"> assertEquals(1, region.getStoreFileList(new byte[][] { b }).size(), "Region did not flush?");</span>
+<span class="source-line-no">216</span><span id="line-216"></span>
+<span class="source-line-no">217</span><span id="line-217"> // now check the region's unflushed seqIds.</span>
+<span class="source-line-no">218</span><span id="line-218"> long seqId = AbstractTestFSWAL.getEarliestMemStoreSeqNum(log, hri.getEncodedNameAsBytes());</span>
+<span class="source-line-no">219</span><span id="line-219"> assertEquals(HConstants.NO_SEQNUM, seqId,</span>
+<span class="source-line-no">220</span><span id="line-220"> "Found seqId for the region which is already flushed");</span>
+<span class="source-line-no">221</span><span id="line-221"></span>
+<span class="source-line-no">222</span><span id="line-222"> region.close();</span>
+<span class="source-line-no">223</span><span id="line-223"> }</span>
+<span class="source-line-no">224</span><span id="line-224"> }</span>
+<span class="source-line-no">225</span><span id="line-225"></span>
+<span class="source-line-no">226</span><span id="line-226"> /**</span>
+<span class="source-line-no">227</span><span id="line-227"> * Regression test for HBASE-30346: FSHLog#getPipeline() must never return null, even when the</span>
+<span class="source-line-no">228</span><span id="line-228"> * underlying DFSOutputStream#getPipeline() legitimately returns null (e.g. the DFS streamer is</span>
+<span class="source-line-no">229</span><span id="line-229"> * closed, or no block pipeline is currently established -- see HDFS-826 and the</span>
+<span class="source-line-no">230</span><span id="line-230"> * DFSOutputStream#getPipeline() javadoc: "returns the list of targets, if any"). Prior to this</span>
+<span class="source-line-no">231</span><span id="line-231"> * fix, AbstractFSWAL#rollWriterInternal's debug-log statement called</span>
+<span class="source-line-no">232</span><span id="line-232"> * Arrays.stream(getPipeline()), which threw a NullPointerException whenever this happened during</span>
+<span class="source-line-no">233</span><span id="line-233"> * a WAL roll, aborting the RegionServer on what was otherwise a successful roll.</span>
+<span class="source-line-no">234</span><span id="line-234"> */</span>
+<span class="source-line-no">235</span><span id="line-235"> @Test</span>
+<span class="source-line-no">236</span><span id="line-236"> public void testGetPipelineDoesNotReturnNullWhenUnderlyingStreamerHasNone() throws Exception {</span>
+<span class="source-line-no">237</span><span id="line-237"> FS.mkdirs(new Path(CommonFSUtils.getRootDir(CONF), this.name));</span>
+<span class="source-line-no">238</span><span id="line-238"> try (FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), this.name,</span>
+<span class="source-line-no">239</span><span id="line-239"> HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null)) {</span>
+<span class="source-line-no">240</span><span id="line-240"> log.init();</span>
+<span class="source-line-no">241</span><span id="line-241"></span>
+<span class="source-line-no">242</span><span id="line-242"> // Simulate the legitimate HDFS contract: the wrapped DFSOutputStream currently has no</span>
+<span class="source-line-no">243</span><span id="line-243"> // established pipeline (e.g. streamer closed, or between blocks) and returns null.</span>
+<span class="source-line-no">244</span><span id="line-244"> DFSOutputStream mockDfsOut = mock(DFSOutputStream.class);</span>
+<span class="source-line-no">245</span><span id="line-245"> when(mockDfsOut.getPipeline()).thenReturn(null);</span>
+<span class="source-line-no">246</span><span id="line-246"> FSDataOutputStream wrappedOut = new HdfsDataOutputStream(mockDfsOut, null);</span>
+<span class="source-line-no">247</span><span id="line-247"></span>
+<span class="source-line-no">248</span><span id="line-248"> Field hdfsOutField = FSHLog.class.getDeclaredField("hdfs_out");</span>
+<span class="source-line-no">249</span><span id="line-249"> hdfsOutField.setAccessible(true);</span>
+<span class="source-line-no">250</span><span id="line-250"> hdfsOutField.set(log, wrappedOut);</span>
+<span class="source-line-no">251</span><span id="line-251"></span>
+<span class="source-line-no">252</span><span id="line-252"> DatanodeInfo[] pipeline = log.getPipeline();</span>
+<span class="source-line-no">253</span><span id="line-253"> assertNotNull(pipeline, "getPipeline() must never return null");</span>
+<span class="source-line-no">254</span><span id="line-254"> assertEquals(0, pipeline.length,</span>
+<span class="source-line-no">255</span><span id="line-255"> "Should normalize a null underlying pipeline to an empty array");</span>
+<span class="source-line-no">256</span><span id="line-256"> }</span>
+<span class="source-line-no">257</span><span id="line-257"> }</span>
+<span class="source-line-no">258</span><span id="line-258">}</span>