You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mi...@apache.org on 2015/12/14 17:38:19 UTC
[08/45] hbase-site git commit: Published site at
04622254f7209c5cfeadcfa137a97fbed161075a.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/736088b3/xref/org/apache/hadoop/hbase/regionserver/HStore.html
----------------------------------------------------------------------
diff --git a/xref/org/apache/hadoop/hbase/regionserver/HStore.html b/xref/org/apache/hadoop/hbase/regionserver/HStore.html
index 1a9620d..929329a 100644
--- a/xref/org/apache/hadoop/hbase/regionserver/HStore.html
+++ b/xref/org/apache/hadoop/hbase/regionserver/HStore.html
@@ -1052,7 +1052,7 @@
<a class="jxr_linenumber" name="1042" href="#1042">1042</a> <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
<a class="jxr_linenumber" name="1043" href="#1043">1043</a> }
<a class="jxr_linenumber" name="1044" href="#1044">1044</a> <em class="jxr_comment">// notify to be called here - only in case of flushes</em>
-<a class="jxr_linenumber" name="1045" href="#1045">1045</a> notifyChangedReadersObservers();
+<a class="jxr_linenumber" name="1045" href="#1045">1045</a> notifyChangedReadersObservers(sfs);
<a class="jxr_linenumber" name="1046" href="#1046">1046</a> <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
<a class="jxr_linenumber" name="1047" href="#1047">1047</a> <strong class="jxr_keyword">long</strong> totalSize = 0;
<a class="jxr_linenumber" name="1048" href="#1048">1048</a> <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
@@ -1070,9 +1070,9 @@
<a class="jxr_linenumber" name="1060" href="#1060">1060</a> <em class="jxr_comment"> * Notify all observers that set of Readers has changed.</em>
<a class="jxr_linenumber" name="1061" href="#1061">1061</a> <em class="jxr_comment"> * @throws IOException</em>
<a class="jxr_linenumber" name="1062" href="#1062">1062</a> <em class="jxr_comment"> */</em>
-<a class="jxr_linenumber" name="1063" href="#1063">1063</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> notifyChangedReadersObservers() <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1063" href="#1063">1063</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> notifyChangedReadersObservers(List<StoreFile> sfs) <strong class="jxr_keyword">throws</strong> IOException {
<a class="jxr_linenumber" name="1064" href="#1064">1064</a> <strong class="jxr_keyword">for</strong> (ChangedReadersObserver o : <strong class="jxr_keyword">this</strong>.changedReaderObservers) {
-<a class="jxr_linenumber" name="1065" href="#1065">1065</a> o.updateReaders();
+<a class="jxr_linenumber" name="1065" href="#1065">1065</a> o.updateReaders(sfs);
<a class="jxr_linenumber" name="1066" href="#1066">1066</a> }
<a class="jxr_linenumber" name="1067" href="#1067">1067</a> }
<a class="jxr_linenumber" name="1068" href="#1068">1068</a>
@@ -1112,1283 +1112,1307 @@
<a class="jxr_linenumber" name="1102" href="#1102">1102</a> }
<a class="jxr_linenumber" name="1103" href="#1103">1103</a>
<a class="jxr_linenumber" name="1104" href="#1104">1104</a> @Override
-<a class="jxr_linenumber" name="1105" href="#1105">1105</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> addChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
-<a class="jxr_linenumber" name="1106" href="#1106">1106</a> <strong class="jxr_keyword">this</strong>.changedReaderObservers.add(o);
-<a class="jxr_linenumber" name="1107" href="#1107">1107</a> }
-<a class="jxr_linenumber" name="1108" href="#1108">1108</a>
-<a class="jxr_linenumber" name="1109" href="#1109">1109</a> @Override
-<a class="jxr_linenumber" name="1110" href="#1110">1110</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> deleteChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
-<a class="jxr_linenumber" name="1111" href="#1111">1111</a> <em class="jxr_comment">// We don't check if observer present; it may not be (legitimately)</em>
-<a class="jxr_linenumber" name="1112" href="#1112">1112</a> <strong class="jxr_keyword">this</strong>.changedReaderObservers.remove(o);
-<a class="jxr_linenumber" name="1113" href="#1113">1113</a> }
-<a class="jxr_linenumber" name="1114" href="#1114">1114</a>
-<a class="jxr_linenumber" name="1115" href="#1115">1115</a> <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
-<a class="jxr_linenumber" name="1116" href="#1116">1116</a> <em class="jxr_comment">// Compaction</em>
-<a class="jxr_linenumber" name="1117" href="#1117">1117</a> <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
-<a class="jxr_linenumber" name="1118" href="#1118">1118</a>
-<a class="jxr_linenumber" name="1119" href="#1119">1119</a> <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1120" href="#1120">1120</a> <em class="jxr_javadoccomment"> * Compact the StoreFiles. This method may take some time, so the calling</em>
-<a class="jxr_linenumber" name="1121" href="#1121">1121</a> <em class="jxr_javadoccomment"> * thread must be able to block for long periods.</em>
-<a class="jxr_linenumber" name="1122" href="#1122">1122</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1123" href="#1123">1123</a> <em class="jxr_javadoccomment"> * <p>During this time, the Store can work as usual, getting values from</em>
-<a class="jxr_linenumber" name="1124" href="#1124">1124</a> <em class="jxr_javadoccomment"> * StoreFiles and writing new StoreFiles from the memstore.</em>
-<a class="jxr_linenumber" name="1125" href="#1125">1125</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1126" href="#1126">1126</a> <em class="jxr_javadoccomment"> * Existing StoreFiles are not destroyed until the new compacted StoreFile is</em>
-<a class="jxr_linenumber" name="1127" href="#1127">1127</a> <em class="jxr_javadoccomment"> * completely written-out to disk.</em>
-<a class="jxr_linenumber" name="1128" href="#1128">1128</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1129" href="#1129">1129</a> <em class="jxr_javadoccomment"> * <p>The compactLock prevents multiple simultaneous compactions.</em>
-<a class="jxr_linenumber" name="1130" href="#1130">1130</a> <em class="jxr_javadoccomment"> * The structureLock prevents us from interfering with other write operations.</em>
-<a class="jxr_linenumber" name="1131" href="#1131">1131</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1132" href="#1132">1132</a> <em class="jxr_javadoccomment"> * <p>We don't want to hold the structureLock for the whole time, as a compact()</em>
-<a class="jxr_linenumber" name="1133" href="#1133">1133</a> <em class="jxr_javadoccomment"> * can be lengthy and we want to allow cache-flushes during this period.</em>
-<a class="jxr_linenumber" name="1134" href="#1134">1134</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1135" href="#1135">1135</a> <em class="jxr_javadoccomment"> * <p> Compaction event should be idempotent, since there is no IO Fencing for</em>
-<a class="jxr_linenumber" name="1136" href="#1136">1136</a> <em class="jxr_javadoccomment"> * the region directory in hdfs. A region server might still try to complete the</em>
-<a class="jxr_linenumber" name="1137" href="#1137">1137</a> <em class="jxr_javadoccomment"> * compaction after it lost the region. That is why the following events are carefully</em>
-<a class="jxr_linenumber" name="1138" href="#1138">1138</a> <em class="jxr_javadoccomment"> * ordered for a compaction:</em>
-<a class="jxr_linenumber" name="1139" href="#1139">1139</a> <em class="jxr_javadoccomment"> * 1. Compaction writes new files under region/.tmp directory (compaction output)</em>
-<a class="jxr_linenumber" name="1140" href="#1140">1140</a> <em class="jxr_javadoccomment"> * 2. Compaction atomically moves the temporary file under region directory</em>
-<a class="jxr_linenumber" name="1141" href="#1141">1141</a> <em class="jxr_javadoccomment"> * 3. Compaction appends a WAL edit containing the compaction input and output files.</em>
-<a class="jxr_linenumber" name="1142" href="#1142">1142</a> <em class="jxr_javadoccomment"> * Forces sync on WAL.</em>
-<a class="jxr_linenumber" name="1143" href="#1143">1143</a> <em class="jxr_javadoccomment"> * 4. Compaction deletes the input files from the region directory.</em>
-<a class="jxr_linenumber" name="1144" href="#1144">1144</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1145" href="#1145">1145</a> <em class="jxr_javadoccomment"> * Failure conditions are handled like this:</em>
-<a class="jxr_linenumber" name="1146" href="#1146">1146</a> <em class="jxr_javadoccomment"> * - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes</em>
-<a class="jxr_linenumber" name="1147" href="#1147">1147</a> <em class="jxr_javadoccomment"> * the compaction later, it will only write the new data file to the region directory.</em>
-<a class="jxr_linenumber" name="1148" href="#1148">1148</a> <em class="jxr_javadoccomment"> * Since we already have this data, this will be idempotent but we will have a redundant</em>
-<a class="jxr_linenumber" name="1149" href="#1149">1149</a> <em class="jxr_javadoccomment"> * copy of the data.</em>
-<a class="jxr_linenumber" name="1150" href="#1150">1150</a> <em class="jxr_javadoccomment"> * - If RS fails between 2 and 3, the region will have a redundant copy of the data. The</em>
-<a class="jxr_linenumber" name="1151" href="#1151">1151</a> <em class="jxr_javadoccomment"> * RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.</em>
-<a class="jxr_linenumber" name="1152" href="#1152">1152</a> <em class="jxr_javadoccomment"> * - If RS fails after 3, the region region server who opens the region will pick up the</em>
-<a class="jxr_linenumber" name="1153" href="#1153">1153</a> <em class="jxr_javadoccomment"> * the compaction marker from the WAL and replay it by removing the compaction input files.</em>
-<a class="jxr_linenumber" name="1154" href="#1154">1154</a> <em class="jxr_javadoccomment"> * Failed RS can also attempt to delete those files, but the operation will be idempotent</em>
+<a class="jxr_linenumber" name="1105" href="#1105">1105</a> <strong class="jxr_keyword">public</strong> List<KeyValueScanner> getScanners(List<StoreFile> files, <strong class="jxr_keyword">boolean</strong> cacheBlocks,
+<a class="jxr_linenumber" name="1106" href="#1106">1106</a> <strong class="jxr_keyword">boolean</strong> isGet, <strong class="jxr_keyword">boolean</strong> usePread, <strong class="jxr_keyword">boolean</strong> isCompaction, <a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.html">ScanQueryMatcher</a> matcher,
+<a class="jxr_linenumber" name="1107" href="#1107">1107</a> byte[] startRow, byte[] stopRow, <strong class="jxr_keyword">long</strong> readPt, <strong class="jxr_keyword">boolean</strong> includeMemstoreScanner) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1108" href="#1108">1108</a> List<KeyValueScanner> memStoreScanners = <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1109" href="#1109">1109</a> <strong class="jxr_keyword">if</strong> (includeMemstoreScanner) {
+<a class="jxr_linenumber" name="1110" href="#1110">1110</a> <strong class="jxr_keyword">this</strong>.lock.readLock().lock();
+<a class="jxr_linenumber" name="1111" href="#1111">1111</a> <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1112" href="#1112">1112</a> memStoreScanners = <strong class="jxr_keyword">this</strong>.memstore.getScanners(readPt);
+<a class="jxr_linenumber" name="1113" href="#1113">1113</a> } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1114" href="#1114">1114</a> <strong class="jxr_keyword">this</strong>.lock.readLock().unlock();
+<a class="jxr_linenumber" name="1115" href="#1115">1115</a> }
+<a class="jxr_linenumber" name="1116" href="#1116">1116</a> }
+<a class="jxr_linenumber" name="1117" href="#1117">1117</a> List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(files,
+<a class="jxr_linenumber" name="1118" href="#1118">1118</a> cacheBlocks, usePread, isCompaction, false, matcher, readPt, isPrimaryReplicaStore());
+<a class="jxr_linenumber" name="1119" href="#1119">1119</a> List<KeyValueScanner> scanners = <strong class="jxr_keyword">new</strong> ArrayList<KeyValueScanner>(sfScanners.size() + 1);
+<a class="jxr_linenumber" name="1120" href="#1120">1120</a> scanners.addAll(sfScanners);
+<a class="jxr_linenumber" name="1121" href="#1121">1121</a> <em class="jxr_comment">// Then the memstore scanners</em>
+<a class="jxr_linenumber" name="1122" href="#1122">1122</a> <strong class="jxr_keyword">if</strong> (memStoreScanners != <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1123" href="#1123">1123</a> scanners.addAll(memStoreScanners);
+<a class="jxr_linenumber" name="1124" href="#1124">1124</a> }
+<a class="jxr_linenumber" name="1125" href="#1125">1125</a> <strong class="jxr_keyword">return</strong> scanners;
+<a class="jxr_linenumber" name="1126" href="#1126">1126</a> }
+<a class="jxr_linenumber" name="1127" href="#1127">1127</a>
+<a class="jxr_linenumber" name="1128" href="#1128">1128</a> @Override
+<a class="jxr_linenumber" name="1129" href="#1129">1129</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> addChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
+<a class="jxr_linenumber" name="1130" href="#1130">1130</a> <strong class="jxr_keyword">this</strong>.changedReaderObservers.add(o);
+<a class="jxr_linenumber" name="1131" href="#1131">1131</a> }
+<a class="jxr_linenumber" name="1132" href="#1132">1132</a>
+<a class="jxr_linenumber" name="1133" href="#1133">1133</a> @Override
+<a class="jxr_linenumber" name="1134" href="#1134">1134</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> deleteChangedReaderObserver(<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChangedReadersObserver.html">ChangedReadersObserver</a> o) {
+<a class="jxr_linenumber" name="1135" href="#1135">1135</a> <em class="jxr_comment">// We don't check if observer present; it may not be (legitimately)</em>
+<a class="jxr_linenumber" name="1136" href="#1136">1136</a> <strong class="jxr_keyword">this</strong>.changedReaderObservers.remove(o);
+<a class="jxr_linenumber" name="1137" href="#1137">1137</a> }
+<a class="jxr_linenumber" name="1138" href="#1138">1138</a>
+<a class="jxr_linenumber" name="1139" href="#1139">1139</a> <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
+<a class="jxr_linenumber" name="1140" href="#1140">1140</a> <em class="jxr_comment">// Compaction</em>
+<a class="jxr_linenumber" name="1141" href="#1141">1141</a> <em class="jxr_comment">//////////////////////////////////////////////////////////////////////////////</em>
+<a class="jxr_linenumber" name="1142" href="#1142">1142</a>
+<a class="jxr_linenumber" name="1143" href="#1143">1143</a> <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1144" href="#1144">1144</a> <em class="jxr_javadoccomment"> * Compact the StoreFiles. This method may take some time, so the calling</em>
+<a class="jxr_linenumber" name="1145" href="#1145">1145</a> <em class="jxr_javadoccomment"> * thread must be able to block for long periods.</em>
+<a class="jxr_linenumber" name="1146" href="#1146">1146</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1147" href="#1147">1147</a> <em class="jxr_javadoccomment"> * <p>During this time, the Store can work as usual, getting values from</em>
+<a class="jxr_linenumber" name="1148" href="#1148">1148</a> <em class="jxr_javadoccomment"> * StoreFiles and writing new StoreFiles from the memstore.</em>
+<a class="jxr_linenumber" name="1149" href="#1149">1149</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1150" href="#1150">1150</a> <em class="jxr_javadoccomment"> * Existing StoreFiles are not destroyed until the new compacted StoreFile is</em>
+<a class="jxr_linenumber" name="1151" href="#1151">1151</a> <em class="jxr_javadoccomment"> * completely written-out to disk.</em>
+<a class="jxr_linenumber" name="1152" href="#1152">1152</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1153" href="#1153">1153</a> <em class="jxr_javadoccomment"> * <p>The compactLock prevents multiple simultaneous compactions.</em>
+<a class="jxr_linenumber" name="1154" href="#1154">1154</a> <em class="jxr_javadoccomment"> * The structureLock prevents us from interfering with other write operations.</em>
<a class="jxr_linenumber" name="1155" href="#1155">1155</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1156" href="#1156">1156</a> <em class="jxr_javadoccomment"> * See HBASE-2231 for details.</em>
-<a class="jxr_linenumber" name="1157" href="#1157">1157</a> <em class="jxr_javadoccomment"> *</em>
-<a class="jxr_linenumber" name="1158" href="#1158">1158</a> <em class="jxr_javadoccomment"> * @param compaction compaction details obtained from requestCompaction()</em>
-<a class="jxr_linenumber" name="1159" href="#1159">1159</a> <em class="jxr_javadoccomment"> * @throws IOException</em>
-<a class="jxr_linenumber" name="1160" href="#1160">1160</a> <em class="jxr_javadoccomment"> * @return Storefile we compacted into or null if we failed or opted out early.</em>
-<a class="jxr_linenumber" name="1161" href="#1161">1161</a> <em class="jxr_javadoccomment"> */</em>
-<a class="jxr_linenumber" name="1162" href="#1162">1162</a> @Override
-<a class="jxr_linenumber" name="1163" href="#1163">1163</a> <strong class="jxr_keyword">public</strong> List<StoreFile> compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
-<a class="jxr_linenumber" name="1164" href="#1164">1164</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1165" href="#1165">1165</a> <strong class="jxr_keyword">return</strong> compact(compaction, throughputController, <strong class="jxr_keyword">null</strong>);
-<a class="jxr_linenumber" name="1166" href="#1166">1166</a> }
-<a class="jxr_linenumber" name="1167" href="#1167">1167</a>
-<a class="jxr_linenumber" name="1168" href="#1168">1168</a> @Override
-<a class="jxr_linenumber" name="1169" href="#1169">1169</a> <strong class="jxr_keyword">public</strong> List<StoreFile> compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
-<a class="jxr_linenumber" name="1170" href="#1170">1170</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1171" href="#1171">1171</a> assert compaction != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1172" href="#1172">1172</a> List<StoreFile> sfs = <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1173" href="#1173">1173</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr = compaction.getRequest();
-<a class="jxr_linenumber" name="1174" href="#1174">1174</a> <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1175" href="#1175">1175</a> <em class="jxr_comment">// Do all sanity checking in here if we have a valid CompactionRequest</em>
-<a class="jxr_linenumber" name="1176" href="#1176">1176</a> <em class="jxr_comment">// because we need to clean up after it on the way out in a finally</em>
-<a class="jxr_linenumber" name="1177" href="#1177">1177</a> <em class="jxr_comment">// block below</em>
-<a class="jxr_linenumber" name="1178" href="#1178">1178</a> <strong class="jxr_keyword">long</strong> compactionStartTime = EnvironmentEdgeManager.currentTime();
-<a class="jxr_linenumber" name="1179" href="#1179">1179</a> assert compaction.hasSelection();
-<a class="jxr_linenumber" name="1180" href="#1180">1180</a> Collection<StoreFile> filesToCompact = cr.getFiles();
-<a class="jxr_linenumber" name="1181" href="#1181">1181</a> assert !filesToCompact.isEmpty();
-<a class="jxr_linenumber" name="1182" href="#1182">1182</a> <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
-<a class="jxr_linenumber" name="1183" href="#1183">1183</a> <em class="jxr_comment">// sanity check: we're compacting files that this store knows about</em>
-<a class="jxr_linenumber" name="1184" href="#1184">1184</a> <em class="jxr_comment">// TODO: change this to LOG.error() after more debugging</em>
-<a class="jxr_linenumber" name="1185" href="#1185">1185</a> Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
-<a class="jxr_linenumber" name="1186" href="#1186">1186</a> }
-<a class="jxr_linenumber" name="1187" href="#1187">1187</a>
-<a class="jxr_linenumber" name="1188" href="#1188">1188</a> <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
-<a class="jxr_linenumber" name="1189" href="#1189">1189</a> LOG.info(<span class="jxr_string">"Starting compaction of "</span> + filesToCompact.size() + <span class="jxr_string">" file(s) in "</span>
-<a class="jxr_linenumber" name="1190" href="#1190">1190</a> + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString()
-<a class="jxr_linenumber" name="1191" href="#1191">1191</a> + <span class="jxr_string">" into tmpdir="</span> + fs.getTempDir() + <span class="jxr_string">", totalSize="</span>
-<a class="jxr_linenumber" name="1192" href="#1192">1192</a> + TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(cr.getSize(), <span class="jxr_string">""</span>, 1));
-<a class="jxr_linenumber" name="1193" href="#1193">1193</a>
-<a class="jxr_linenumber" name="1194" href="#1194">1194</a> <em class="jxr_comment">// Commence the compaction.</em>
-<a class="jxr_linenumber" name="1195" href="#1195">1195</a> List<Path> newFiles = compaction.compact(throughputController, user);
-<a class="jxr_linenumber" name="1196" href="#1196">1196</a>
-<a class="jxr_linenumber" name="1197" href="#1197">1197</a> <em class="jxr_comment">// TODO: get rid of this!</em>
-<a class="jxr_linenumber" name="1198" href="#1198">1198</a> <strong class="jxr_keyword">if</strong> (!<strong class="jxr_keyword">this</strong>.conf.getBoolean(<span class="jxr_string">"hbase.hstore.compaction.complete"</span>, <strong class="jxr_keyword">true</strong>)) {
-<a class="jxr_linenumber" name="1199" href="#1199">1199</a> LOG.warn(<span class="jxr_string">"hbase.hstore.compaction.complete is set to false"</span>);
-<a class="jxr_linenumber" name="1200" href="#1200">1200</a> sfs = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(newFiles.size());
-<a class="jxr_linenumber" name="1201" href="#1201">1201</a> <strong class="jxr_keyword">final</strong> <strong class="jxr_keyword">boolean</strong> evictOnClose =
-<a class="jxr_linenumber" name="1202" href="#1202">1202</a> cacheConf != <strong class="jxr_keyword">null</strong>? cacheConf.shouldEvictOnClose(): <strong class="jxr_keyword">true</strong>;
-<a class="jxr_linenumber" name="1203" href="#1203">1203</a> <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
-<a class="jxr_linenumber" name="1204" href="#1204">1204</a> <em class="jxr_comment">// Create storefile around what we wrote with a reader on it.</em>
-<a class="jxr_linenumber" name="1205" href="#1205">1205</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = createStoreFileAndReader(newFile);
-<a class="jxr_linenumber" name="1206" href="#1206">1206</a> sf.closeReader(evictOnClose);
-<a class="jxr_linenumber" name="1207" href="#1207">1207</a> sfs.add(sf);
-<a class="jxr_linenumber" name="1208" href="#1208">1208</a> }
-<a class="jxr_linenumber" name="1209" href="#1209">1209</a> <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1156" href="#1156">1156</a> <em class="jxr_javadoccomment"> * <p>We don't want to hold the structureLock for the whole time, as a compact()</em>
+<a class="jxr_linenumber" name="1157" href="#1157">1157</a> <em class="jxr_javadoccomment"> * can be lengthy and we want to allow cache-flushes during this period.</em>
+<a class="jxr_linenumber" name="1158" href="#1158">1158</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1159" href="#1159">1159</a> <em class="jxr_javadoccomment"> * <p> Compaction event should be idempotent, since there is no IO Fencing for</em>
+<a class="jxr_linenumber" name="1160" href="#1160">1160</a> <em class="jxr_javadoccomment"> * the region directory in hdfs. A region server might still try to complete the</em>
+<a class="jxr_linenumber" name="1161" href="#1161">1161</a> <em class="jxr_javadoccomment"> * compaction after it lost the region. That is why the following events are carefully</em>
+<a class="jxr_linenumber" name="1162" href="#1162">1162</a> <em class="jxr_javadoccomment"> * ordered for a compaction:</em>
+<a class="jxr_linenumber" name="1163" href="#1163">1163</a> <em class="jxr_javadoccomment"> * 1. Compaction writes new files under region/.tmp directory (compaction output)</em>
+<a class="jxr_linenumber" name="1164" href="#1164">1164</a> <em class="jxr_javadoccomment"> * 2. Compaction atomically moves the temporary file under region directory</em>
+<a class="jxr_linenumber" name="1165" href="#1165">1165</a> <em class="jxr_javadoccomment"> * 3. Compaction appends a WAL edit containing the compaction input and output files.</em>
+<a class="jxr_linenumber" name="1166" href="#1166">1166</a> <em class="jxr_javadoccomment"> * Forces sync on WAL.</em>
+<a class="jxr_linenumber" name="1167" href="#1167">1167</a> <em class="jxr_javadoccomment"> * 4. Compaction deletes the input files from the region directory.</em>
+<a class="jxr_linenumber" name="1168" href="#1168">1168</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1169" href="#1169">1169</a> <em class="jxr_javadoccomment"> * Failure conditions are handled like this:</em>
+<a class="jxr_linenumber" name="1170" href="#1170">1170</a> <em class="jxr_javadoccomment"> * - If RS fails before 2, compaction wont complete. Even if RS lives on and finishes</em>
+<a class="jxr_linenumber" name="1171" href="#1171">1171</a> <em class="jxr_javadoccomment"> * the compaction later, it will only write the new data file to the region directory.</em>
+<a class="jxr_linenumber" name="1172" href="#1172">1172</a> <em class="jxr_javadoccomment"> * Since we already have this data, this will be idempotent but we will have a redundant</em>
+<a class="jxr_linenumber" name="1173" href="#1173">1173</a> <em class="jxr_javadoccomment"> * copy of the data.</em>
+<a class="jxr_linenumber" name="1174" href="#1174">1174</a> <em class="jxr_javadoccomment"> * - If RS fails between 2 and 3, the region will have a redundant copy of the data. The</em>
+<a class="jxr_linenumber" name="1175" href="#1175">1175</a> <em class="jxr_javadoccomment"> * RS that failed won't be able to finish snyc() for WAL because of lease recovery in WAL.</em>
+<a class="jxr_linenumber" name="1176" href="#1176">1176</a> <em class="jxr_javadoccomment"> * - If RS fails after 3, the region region server who opens the region will pick up the</em>
+<a class="jxr_linenumber" name="1177" href="#1177">1177</a> <em class="jxr_javadoccomment"> * the compaction marker from the WAL and replay it by removing the compaction input files.</em>
+<a class="jxr_linenumber" name="1178" href="#1178">1178</a> <em class="jxr_javadoccomment"> * Failed RS can also attempt to delete those files, but the operation will be idempotent</em>
+<a class="jxr_linenumber" name="1179" href="#1179">1179</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1180" href="#1180">1180</a> <em class="jxr_javadoccomment"> * See HBASE-2231 for details.</em>
+<a class="jxr_linenumber" name="1181" href="#1181">1181</a> <em class="jxr_javadoccomment"> *</em>
+<a class="jxr_linenumber" name="1182" href="#1182">1182</a> <em class="jxr_javadoccomment"> * @param compaction compaction details obtained from requestCompaction()</em>
+<a class="jxr_linenumber" name="1183" href="#1183">1183</a> <em class="jxr_javadoccomment"> * @throws IOException</em>
+<a class="jxr_linenumber" name="1184" href="#1184">1184</a> <em class="jxr_javadoccomment"> * @return Storefile we compacted into or null if we failed or opted out early.</em>
+<a class="jxr_linenumber" name="1185" href="#1185">1185</a> <em class="jxr_javadoccomment"> */</em>
+<a class="jxr_linenumber" name="1186" href="#1186">1186</a> @Override
+<a class="jxr_linenumber" name="1187" href="#1187">1187</a> <strong class="jxr_keyword">public</strong> List<StoreFile> compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
+<a class="jxr_linenumber" name="1188" href="#1188">1188</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1189" href="#1189">1189</a> <strong class="jxr_keyword">return</strong> compact(compaction, throughputController, <strong class="jxr_keyword">null</strong>);
+<a class="jxr_linenumber" name="1190" href="#1190">1190</a> }
+<a class="jxr_linenumber" name="1191" href="#1191">1191</a>
+<a class="jxr_linenumber" name="1192" href="#1192">1192</a> @Override
+<a class="jxr_linenumber" name="1193" href="#1193">1193</a> <strong class="jxr_keyword">public</strong> List<StoreFile> compact(<a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionContext.html">CompactionContext</a> compaction,
+<a class="jxr_linenumber" name="1194" href="#1194">1194</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionThroughputController.html">CompactionThroughputController</a> throughputController, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1195" href="#1195">1195</a> assert compaction != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1196" href="#1196">1196</a> List<StoreFile> sfs = <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1197" href="#1197">1197</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr = compaction.getRequest();
+<a class="jxr_linenumber" name="1198" href="#1198">1198</a> <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1199" href="#1199">1199</a> <em class="jxr_comment">// Do all sanity checking in here if we have a valid CompactionRequest</em>
+<a class="jxr_linenumber" name="1200" href="#1200">1200</a> <em class="jxr_comment">// because we need to clean up after it on the way out in a finally</em>
+<a class="jxr_linenumber" name="1201" href="#1201">1201</a> <em class="jxr_comment">// block below</em>
+<a class="jxr_linenumber" name="1202" href="#1202">1202</a> <strong class="jxr_keyword">long</strong> compactionStartTime = EnvironmentEdgeManager.currentTime();
+<a class="jxr_linenumber" name="1203" href="#1203">1203</a> assert compaction.hasSelection();
+<a class="jxr_linenumber" name="1204" href="#1204">1204</a> Collection<StoreFile> filesToCompact = cr.getFiles();
+<a class="jxr_linenumber" name="1205" href="#1205">1205</a> assert !filesToCompact.isEmpty();
+<a class="jxr_linenumber" name="1206" href="#1206">1206</a> <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
+<a class="jxr_linenumber" name="1207" href="#1207">1207</a> <em class="jxr_comment">// sanity check: we're compacting files that this store knows about</em>
+<a class="jxr_linenumber" name="1208" href="#1208">1208</a> <em class="jxr_comment">// TODO: change this to LOG.error() after more debugging</em>
+<a class="jxr_linenumber" name="1209" href="#1209">1209</a> Preconditions.checkArgument(filesCompacting.containsAll(filesToCompact));
<a class="jxr_linenumber" name="1210" href="#1210">1210</a> }
-<a class="jxr_linenumber" name="1211" href="#1211">1211</a> <em class="jxr_comment">// Do the steps necessary to complete the compaction.</em>
-<a class="jxr_linenumber" name="1212" href="#1212">1212</a> sfs = moveCompatedFilesIntoPlace(cr, newFiles, user);
-<a class="jxr_linenumber" name="1213" href="#1213">1213</a> writeCompactionWalRecord(filesToCompact, sfs);
-<a class="jxr_linenumber" name="1214" href="#1214">1214</a> replaceStoreFiles(filesToCompact, sfs);
-<a class="jxr_linenumber" name="1215" href="#1215">1215</a> <strong class="jxr_keyword">if</strong> (cr.isMajor()) {
-<a class="jxr_linenumber" name="1216" href="#1216">1216</a> majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
-<a class="jxr_linenumber" name="1217" href="#1217">1217</a> majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
-<a class="jxr_linenumber" name="1218" href="#1218">1218</a> } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1219" href="#1219">1219</a> compactedCellsCount += getCompactionProgress().totalCompactingKVs;
-<a class="jxr_linenumber" name="1220" href="#1220">1220</a> compactedCellsSize += getCompactionProgress().totalCompactedSize;
-<a class="jxr_linenumber" name="1221" href="#1221">1221</a> }
-<a class="jxr_linenumber" name="1222" href="#1222">1222</a> <em class="jxr_comment">// At this point the store will use new files for all new scanners.</em>
-<a class="jxr_linenumber" name="1223" href="#1223">1223</a> completeCompaction(filesToCompact); <em class="jxr_comment">// update store size.</em>
-<a class="jxr_linenumber" name="1224" href="#1224">1224</a>
-<a class="jxr_linenumber" name="1225" href="#1225">1225</a> logCompactionEndMessage(cr, sfs, compactionStartTime);
-<a class="jxr_linenumber" name="1226" href="#1226">1226</a> <strong class="jxr_keyword">return</strong> sfs;
-<a class="jxr_linenumber" name="1227" href="#1227">1227</a> } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1228" href="#1228">1228</a> finishCompactionRequest(cr);
-<a class="jxr_linenumber" name="1229" href="#1229">1229</a> }
-<a class="jxr_linenumber" name="1230" href="#1230">1230</a> }
-<a class="jxr_linenumber" name="1231" href="#1231">1231</a>
-<a class="jxr_linenumber" name="1232" href="#1232">1232</a> <strong class="jxr_keyword">private</strong> List<StoreFile> moveCompatedFilesIntoPlace(
-<a class="jxr_linenumber" name="1233" href="#1233">1233</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List<Path> newFiles, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1234" href="#1234">1234</a> List<StoreFile> sfs = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(newFiles.size());
-<a class="jxr_linenumber" name="1235" href="#1235">1235</a> <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
-<a class="jxr_linenumber" name="1236" href="#1236">1236</a> assert newFile != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1237" href="#1237">1237</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
-<a class="jxr_linenumber" name="1238" href="#1238">1238</a> <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1239" href="#1239">1239</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/Store.html">Store</a> thisStore = <strong class="jxr_keyword">this</strong>;
-<a class="jxr_linenumber" name="1240" href="#1240">1240</a> <strong class="jxr_keyword">if</strong> (user == <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1241" href="#1241">1241</a> getCoprocessorHost().postCompact(thisStore, sf, cr);
-<a class="jxr_linenumber" name="1242" href="#1242">1242</a> } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1243" href="#1243">1243</a> <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1244" href="#1244">1244</a> user.getUGI().doAs(<strong class="jxr_keyword">new</strong> PrivilegedExceptionAction<Void>() {
-<a class="jxr_linenumber" name="1245" href="#1245">1245</a> @Override
-<a class="jxr_linenumber" name="1246" href="#1246">1246</a> <strong class="jxr_keyword">public</strong> Void run() <strong class="jxr_keyword">throws</strong> Exception {
-<a class="jxr_linenumber" name="1247" href="#1247">1247</a> getCoprocessorHost().postCompact(thisStore, sf, cr);
-<a class="jxr_linenumber" name="1248" href="#1248">1248</a> <strong class="jxr_keyword">return</strong> <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1249" href="#1249">1249</a> }
-<a class="jxr_linenumber" name="1250" href="#1250">1250</a> });
-<a class="jxr_linenumber" name="1251" href="#1251">1251</a> } <strong class="jxr_keyword">catch</strong> (InterruptedException ie) {
-<a class="jxr_linenumber" name="1252" href="#1252">1252</a> InterruptedIOException iioe = <strong class="jxr_keyword">new</strong> InterruptedIOException();
-<a class="jxr_linenumber" name="1253" href="#1253">1253</a> iioe.initCause(ie);
-<a class="jxr_linenumber" name="1254" href="#1254">1254</a> <strong class="jxr_keyword">throw</strong> iioe;
-<a class="jxr_linenumber" name="1255" href="#1255">1255</a> }
-<a class="jxr_linenumber" name="1256" href="#1256">1256</a> }
-<a class="jxr_linenumber" name="1257" href="#1257">1257</a> }
-<a class="jxr_linenumber" name="1258" href="#1258">1258</a> assert sf != <strong class="jxr_keyword">null</strong>;
-<a class="jxr_linenumber" name="1259" href="#1259">1259</a> sfs.add(sf);
-<a class="jxr_linenumber" name="1260" href="#1260">1260</a> }
-<a class="jxr_linenumber" name="1261" href="#1261">1261</a> <strong class="jxr_keyword">return</strong> sfs;
-<a class="jxr_linenumber" name="1262" href="#1262">1262</a> }
-<a class="jxr_linenumber" name="1263" href="#1263">1263</a>
-<a class="jxr_linenumber" name="1264" href="#1264">1264</a> <em class="jxr_comment">// Package-visible for tests</em>
-<a class="jxr_linenumber" name="1265" href="#1265">1265</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> moveFileIntoPlace(<strong class="jxr_keyword">final</strong> Path newFile) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1266" href="#1266">1266</a> validateStoreFile(newFile);
-<a class="jxr_linenumber" name="1267" href="#1267">1267</a> <em class="jxr_comment">// Move the file into the right spot</em>
-<a class="jxr_linenumber" name="1268" href="#1268">1268</a> Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
-<a class="jxr_linenumber" name="1269" href="#1269">1269</a> <strong class="jxr_keyword">return</strong> createStoreFileAndReader(destPath);
-<a class="jxr_linenumber" name="1270" href="#1270">1270</a> }
-<a class="jxr_linenumber" name="1271" href="#1271">1271</a>
-<a class="jxr_linenumber" name="1272" href="#1272">1272</a> <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1273" href="#1273">1273</a> <em class="jxr_javadoccomment"> * Writes the compaction WAL record.</em>
-<a class="jxr_linenumber" name="1274" href="#1274">1274</a> <em class="jxr_javadoccomment"> * @param filesCompacted Files compacted (input).</em>
-<a class="jxr_linenumber" name="1275" href="#1275">1275</a> <em class="jxr_javadoccomment"> * @param newFiles Files from compaction.</em>
-<a class="jxr_linenumber" name="1276" href="#1276">1276</a> <em class="jxr_javadoccomment"> */</em>
-<a class="jxr_linenumber" name="1277" href="#1277">1277</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
-<a class="jxr_linenumber" name="1278" href="#1278">1278</a> Collection<StoreFile> newFiles) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1279" href="#1279">1279</a> <strong class="jxr_keyword">if</strong> (region.getWAL() == <strong class="jxr_keyword">null</strong>) <strong class="jxr_keyword">return</strong>;
-<a class="jxr_linenumber" name="1280" href="#1280">1280</a> List<Path> inputPaths = <strong class="jxr_keyword">new</strong> ArrayList<Path>(filesCompacted.size());
-<a class="jxr_linenumber" name="1281" href="#1281">1281</a> <strong class="jxr_keyword">for</strong> (StoreFile f : filesCompacted) {
-<a class="jxr_linenumber" name="1282" href="#1282">1282</a> inputPaths.add(f.getPath());
-<a class="jxr_linenumber" name="1283" href="#1283">1283</a> }
-<a class="jxr_linenumber" name="1284" href="#1284">1284</a> List<Path> outputPaths = <strong class="jxr_keyword">new</strong> ArrayList<Path>(newFiles.size());
-<a class="jxr_linenumber" name="1285" href="#1285">1285</a> <strong class="jxr_keyword">for</strong> (StoreFile f : newFiles) {
-<a class="jxr_linenumber" name="1286" href="#1286">1286</a> outputPaths.add(f.getPath());
-<a class="jxr_linenumber" name="1287" href="#1287">1287</a> }
-<a class="jxr_linenumber" name="1288" href="#1288">1288</a> <a href="../../../../../org/apache/hadoop/hbase/HRegionInfo.html">HRegionInfo</a> info = <strong class="jxr_keyword">this</strong>.region.getRegionInfo();
-<a class="jxr_linenumber" name="1289" href="#1289">1289</a> CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
-<a class="jxr_linenumber" name="1290" href="#1290">1290</a> family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
-<a class="jxr_linenumber" name="1291" href="#1291">1291</a> WALUtil.writeCompactionMarker(region.getWAL(), <strong class="jxr_keyword">this</strong>.region.getTableDesc(),
-<a class="jxr_linenumber" name="1292" href="#1292">1292</a> <strong class="jxr_keyword">this</strong>.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
-<a class="jxr_linenumber" name="1293" href="#1293">1293</a> }
-<a class="jxr_linenumber" name="1294" href="#1294">1294</a>
-<a class="jxr_linenumber" name="1295" href="#1295">1295</a> @VisibleForTesting
-<a class="jxr_linenumber" name="1296" href="#1296">1296</a> <strong class="jxr_keyword">void</strong> replaceStoreFiles(<strong class="jxr_keyword">final</strong> Collection<StoreFile> compactedFiles,
-<a class="jxr_linenumber" name="1297" href="#1297">1297</a> <strong class="jxr_keyword">final</strong> Collection<StoreFile> result) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1298" href="#1298">1298</a> <strong class="jxr_keyword">this</strong>.lock.writeLock().lock();
-<a class="jxr_linenumber" name="1299" href="#1299">1299</a> <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1300" href="#1300">1300</a> <strong class="jxr_keyword">this</strong>.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
-<a class="jxr_linenumber" name="1301" href="#1301">1301</a> filesCompacting.removeAll(compactedFiles); <em class="jxr_comment">// safe bc: lock.writeLock();</em>
-<a class="jxr_linenumber" name="1302" href="#1302">1302</a> } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1303" href="#1303">1303</a> <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
-<a class="jxr_linenumber" name="1304" href="#1304">1304</a> }
-<a class="jxr_linenumber" name="1305" href="#1305">1305</a> }
-<a class="jxr_linenumber" name="1306" href="#1306">1306</a>
-<a class="jxr_linenumber" name="1307" href="#1307">1307</a> <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1308" href="#1308">1308</a> <em class="jxr_javadoccomment"> * Log a very elaborate compaction completion message.</em>
-<a class="jxr_linenumber" name="1309" href="#1309">1309</a> <em class="jxr_javadoccomment"> * @param cr Request.</em>
-<a class="jxr_linenumber" name="1310" href="#1310">1310</a> <em class="jxr_javadoccomment"> * @param sfs Resulting files.</em>
-<a class="jxr_linenumber" name="1311" href="#1311">1311</a> <em class="jxr_javadoccomment"> * @param compactionStartTime Start time.</em>
-<a class="jxr_linenumber" name="1312" href="#1312">1312</a> <em class="jxr_javadoccomment"> */</em>
-<a class="jxr_linenumber" name="1313" href="#1313">1313</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> logCompactionEndMessage(
-<a class="jxr_linenumber" name="1314" href="#1314">1314</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List<StoreFile> sfs, <strong class="jxr_keyword">long</strong> compactionStartTime) {
-<a class="jxr_linenumber" name="1315" href="#1315">1315</a> <strong class="jxr_keyword">long</strong> now = EnvironmentEdgeManager.currentTime();
-<a class="jxr_linenumber" name="1316" href="#1316">1316</a> StringBuilder message = <strong class="jxr_keyword">new</strong> StringBuilder(
-<a class="jxr_linenumber" name="1317" href="#1317">1317</a> <span class="jxr_string">"Completed"</span> + (cr.isMajor() ? <span class="jxr_string">" major"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" compaction of "</span>
-<a class="jxr_linenumber" name="1318" href="#1318">1318</a> + cr.getFiles().size() + (cr.isAllFiles() ? <span class="jxr_string">" (all)"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" file(s) in "</span>
-<a class="jxr_linenumber" name="1319" href="#1319">1319</a> + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString() + <span class="jxr_string">" into "</span>);
-<a class="jxr_linenumber" name="1320" href="#1320">1320</a> <strong class="jxr_keyword">if</strong> (sfs.isEmpty()) {
-<a class="jxr_linenumber" name="1321" href="#1321">1321</a> message.append(<span class="jxr_string">"none, "</span>);
-<a class="jxr_linenumber" name="1322" href="#1322">1322</a> } <strong class="jxr_keyword">else</strong> {
-<a class="jxr_linenumber" name="1323" href="#1323">1323</a> <strong class="jxr_keyword">for</strong> (StoreFile sf: sfs) {
-<a class="jxr_linenumber" name="1324" href="#1324">1324</a> message.append(sf.getPath().getName());
-<a class="jxr_linenumber" name="1325" href="#1325">1325</a> message.append(<span class="jxr_string">"(size="</span>);
-<a class="jxr_linenumber" name="1326" href="#1326">1326</a> message.append(TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(sf.getReader().length(), <span class="jxr_string">""</span>, 1));
-<a class="jxr_linenumber" name="1327" href="#1327">1327</a> message.append(<span class="jxr_string">"), "</span>);
-<a class="jxr_linenumber" name="1328" href="#1328">1328</a> }
-<a class="jxr_linenumber" name="1329" href="#1329">1329</a> }
-<a class="jxr_linenumber" name="1330" href="#1330">1330</a> message.append(<span class="jxr_string">"total size for store is "</span>)
-<a class="jxr_linenumber" name="1331" href="#1331">1331</a> .append(StringUtils.TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(storeSize, <span class="jxr_string">""</span>, 1))
-<a class="jxr_linenumber" name="1332" href="#1332">1332</a> .append(<span class="jxr_string">". This selection was in queue for "</span>)
-<a class="jxr_linenumber" name="1333" href="#1333">1333</a> .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
-<a class="jxr_linenumber" name="1334" href="#1334">1334</a> .append(<span class="jxr_string">", and took "</span>).append(StringUtils.formatTimeDiff(now, compactionStartTime))
-<a class="jxr_linenumber" name="1335" href="#1335">1335</a> .append(<span class="jxr_string">" to execute."</span>);
-<a class="jxr_linenumber" name="1336" href="#1336">1336</a> LOG.info(message.toString());
-<a class="jxr_linenumber" name="1337" href="#1337">1337</a> <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
-<a class="jxr_linenumber" name="1338" href="#1338">1338</a> <strong class="jxr_keyword">int</strong> fileCount = storeEngine.getStoreFileManager().getStorefileCount();
-<a class="jxr_linenumber" name="1339" href="#1339">1339</a> <strong class="jxr_keyword">long</strong> resultSize = 0;
-<a class="jxr_linenumber" name="1340" href="#1340">1340</a> <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
-<a class="jxr_linenumber" name="1341" href="#1341">1341</a> resultSize += sf.getReader().length();
-<a class="jxr_linenumber" name="1342" href="#1342">1342</a> }
-<a class="jxr_linenumber" name="1343" href="#1343">1343</a> String traceMessage = <span class="jxr_string">"COMPACTION start,end,size out,files in,files out,store size,"</span>
-<a class="jxr_linenumber" name="1344" href="#1344">1344</a> + <span class="jxr_string">"store files ["</span> + compactionStartTime + <span class="jxr_string">","</span> + now + <span class="jxr_string">","</span> + resultSize + <span class="jxr_string">","</span>
-<a class="jxr_linenumber" name="1345" href="#1345">1345</a> + cr.getFiles().size() + <span class="jxr_string">","</span> + sfs.size() + <span class="jxr_string">","</span> + storeSize + <span class="jxr_string">","</span> + fileCount + <span class="jxr_string">"]"</span>;
-<a class="jxr_linenumber" name="1346" href="#1346">1346</a> LOG.trace(traceMessage);
-<a class="jxr_linenumber" name="1347" href="#1347">1347</a> }
-<a class="jxr_linenumber" name="1348" href="#1348">1348</a> }
-<a class="jxr_linenumber" name="1349" href="#1349">1349</a>
-<a class="jxr_linenumber" name="1350" href="#1350">1350</a> <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1351" href="#1351">1351</a> <em class="jxr_javadoccomment"> * Call to complete a compaction. Its for the case where we find in the WAL a compaction</em>
-<a class="jxr_linenumber" name="1352" href="#1352">1352</a> <em class="jxr_javadoccomment"> * that was not finished. We could find one recovering a WAL after a regionserver crash.</em>
-<a class="jxr_linenumber" name="1353" href="#1353">1353</a> <em class="jxr_javadoccomment"> * See HBASE-2231.</em>
-<a class="jxr_linenumber" name="1354" href="#1354">1354</a> <em class="jxr_javadoccomment"> * @param compaction</em>
-<a class="jxr_linenumber" name="1355" href="#1355">1355</a> <em class="jxr_javadoccomment"> */</em>
-<a class="jxr_linenumber" name="1356" href="#1356">1356</a> @Override
-<a class="jxr_linenumber" name="1357" href="#1357">1357</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> replayCompactionMarker(CompactionDescriptor compaction,
-<a class="jxr_linenumber" name="1358" href="#1358">1358</a> <strong class="jxr_keyword">boolean</strong> pickCompactionFiles, <strong class="jxr_keyword">boolean</strong> removeFiles)
-<a class="jxr_linenumber" name="1359" href="#1359">1359</a> <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1360" href="#1360">1360</a> LOG.debug(<span class="jxr_string">"Completing compaction from the WAL marker"</span>);
-<a class="jxr_linenumber" name="1361" href="#1361">1361</a> List<String> compactionInputs = compaction.getCompactionInputList();
-<a class="jxr_linenumber" name="1362" href="#1362">1362</a> List<String> compactionOutputs = Lists.newArrayList(compaction.getCompactionOutputList());
-<a class="jxr_linenumber" name="1363" href="#1363">1363</a>
-<a class="jxr_linenumber" name="1364" href="#1364">1364</a> <em class="jxr_comment">// The Compaction Marker is written after the compaction is completed,</em>
-<a class="jxr_linenumber" name="1365" href="#1365">1365</a> <em class="jxr_comment">// and the files moved into the region/family folder.</em>
-<a class="jxr_linenumber" name="1366" href="#1366">1366</a> <em class="jxr_comment">//</em>
-<a class="jxr_linenumber" name="1367" href="#1367">1367</a> <em class="jxr_comment">// If we crash after the entry is written, we may not have removed the</em>
-<a class="jxr_linenumber" name="1368" href="#1368">1368</a> <em class="jxr_comment">// input files, but the output file is present.</em>
-<a class="jxr_linenumber" name="1369" href="#1369">1369</a> <em class="jxr_comment">// (The unremoved input files will be removed by this function)</em>
-<a class="jxr_linenumber" name="1370" href="#1370">1370</a> <em class="jxr_comment">//</em>
-<a class="jxr_linenumber" name="1371" href="#1371">1371</a> <em class="jxr_comment">// If we scan the directory and the file is not present, it can mean that:</em>
-<a class="jxr_linenumber" name="1372" href="#1372">1372</a> <em class="jxr_comment">// - The file was manually removed by the user</em>
-<a class="jxr_linenumber" name="1373" href="#1373">1373</a> <em class="jxr_comment">// - The file was removed as consequence of subsequent compaction</em>
-<a class="jxr_linenumber" name="1374" href="#1374">1374</a> <em class="jxr_comment">// so, we can't do anything with the "compaction output list" because those</em>
-<a class="jxr_linenumber" name="1375" href="#1375">1375</a> <em class="jxr_comment">// files have already been loaded when opening the region (by virtue of</em>
-<a class="jxr_linenumber" name="1376" href="#1376">1376</a> <em class="jxr_comment">// being in the store's folder) or they may be missing due to a compaction.</em>
-<a class="jxr_linenumber" name="1377" href="#1377">1377</a>
-<a class="jxr_linenumber" name="1378" href="#1378">1378</a> String familyName = <strong class="jxr_keyword">this</strong>.getColumnFamilyName();
-<a class="jxr_linenumber" name="1379" href="#1379">1379</a> List<String> inputFiles = <strong class="jxr_keyword">new</strong> ArrayList<String>(compactionInputs.size());
-<a class="jxr_linenumber" name="1380" href="#1380">1380</a> <strong class="jxr_keyword">for</strong> (String compactionInput : compactionInputs) {
-<a class="jxr_linenumber" name="1381" href="#1381">1381</a> Path inputPath = fs.getStoreFilePath(familyName, compactionInput);
-<a class="jxr_linenumber" name="1382" href="#1382">1382</a> inputFiles.add(inputPath.getName());
-<a class="jxr_linenumber" name="1383" href="#1383">1383</a> }
-<a class="jxr_linenumber" name="1384" href="#1384">1384</a>
-<a class="jxr_linenumber" name="1385" href="#1385">1385</a> <em class="jxr_comment">//some of the input files might already be deleted</em>
-<a class="jxr_linenumber" name="1386" href="#1386">1386</a> List<StoreFile> inputStoreFiles = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(compactionInputs.size());
-<a class="jxr_linenumber" name="1387" href="#1387">1387</a> <strong class="jxr_keyword">for</strong> (StoreFile sf : <strong class="jxr_keyword">this</strong>.getStorefiles()) {
-<a class="jxr_linenumber" name="1388" href="#1388">1388</a> <strong class="jxr_keyword">if</strong> (inputFiles.contains(sf.getPath().getName())) {
-<a class="jxr_linenumber" name="1389" href="#1389">1389</a> inputStoreFiles.add(sf);
-<a class="jxr_linenumber" name="1390" href="#1390">1390</a> }
-<a class="jxr_linenumber" name="1391" href="#1391">1391</a> }
-<a class="jxr_linenumber" name="1392" href="#1392">1392</a>
-<a class="jxr_linenumber" name="1393" href="#1393">1393</a> <em class="jxr_comment">// check whether we need to pick up the new files</em>
-<a class="jxr_linenumber" name="1394" href="#1394">1394</a> List<StoreFile> outputStoreFiles = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(compactionOutputs.size());
-<a class="jxr_linenumber" name="1395" href="#1395">1395</a>
-<a class="jxr_linenumber" name="1396" href="#1396">1396</a> <strong class="jxr_keyword">if</strong> (pickCompactionFiles) {
-<a class="jxr_linenumber" name="1397" href="#1397">1397</a> <strong class="jxr_keyword">for</strong> (StoreFile sf : <strong class="jxr_keyword">this</strong>.getStorefiles()) {
-<a class="jxr_linenumber" name="1398" href="#1398">1398</a> compactionOutputs.remove(sf.getPath().getName());
-<a class="jxr_linenumber" name="1399" href="#1399">1399</a> }
-<a class="jxr_linenumber" name="1400" href="#1400">1400</a> <strong class="jxr_keyword">for</strong> (String compactionOutput : compactionOutputs) {
-<a class="jxr_linenumber" name="1401" href="#1401">1401</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFileInfo.html">StoreFileInfo</a> storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), compactionOutput);
-<a class="jxr_linenumber" name="1402" href="#1402">1402</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> storeFile = createStoreFileAndReader(storeFileInfo);
-<a class="jxr_linenumber" name="1403" href="#1403">1403</a> outputStoreFiles.add(storeFile);
-<a class="jxr_linenumber" name="1404" href="#1404">1404</a> }
-<a class="jxr_linenumber" name="1405" href="#1405">1405</a> }
-<a class="jxr_linenumber" name="1406" href="#1406">1406</a>
-<a class="jxr_linenumber" name="1407" href="#1407">1407</a> <strong class="jxr_keyword">if</strong> (!inputStoreFiles.isEmpty() || !outputStoreFiles.isEmpty()) {
-<a class="jxr_linenumber" name="1408" href="#1408">1408</a> LOG.info(<span class="jxr_string">"Replaying compaction marker, replacing input files: "</span> +
-<a class="jxr_linenumber" name="1409" href="#1409">1409</a> inputStoreFiles + <span class="jxr_string">" with output files : "</span> + outputStoreFiles);
-<a class="jxr_linenumber" name="1410" href="#1410">1410</a> <strong class="jxr_keyword">this</strong>.replaceStoreFiles(inputStoreFiles, outputStoreFiles);
-<a class="jxr_linenumber" name="1411" href="#1411">1411</a> <strong class="jxr_keyword">this</strong>.completeCompaction(inputStoreFiles);
-<a class="jxr_linenumber" name="1412" href="#1412">1412</a> }
-<a class="jxr_linenumber" name="1413" href="#1413">1413</a> }
-<a class="jxr_linenumber" name="1414" href="#1414">1414</a>
-<a class="jxr_linenumber" name="1415" href="#1415">1415</a> <em class="jxr_javadoccomment">/**</em>
-<a class="jxr_linenumber" name="1416" href="#1416">1416</a> <em class="jxr_javadoccomment"> * This method tries to compact N recent files for testing.</em>
-<a class="jxr_linenumber" name="1417" href="#1417">1417</a> <em class="jxr_javadoccomment"> * Note that because compacting "recent" files only makes sense for some policies,</em>
-<a class="jxr_linenumber" name="1418" href="#1418">1418</a> <em class="jxr_javadoccomment"> * e.g. the default one, it assumes default policy is used. It doesn't use policy,</em>
-<a class="jxr_linenumber" name="1419" href="#1419">1419</a> <em class="jxr_javadoccomment"> * but instead makes a compaction candidate list by itself.</em>
-<a class="jxr_linenumber" name="1420" href="#1420">1420</a> <em class="jxr_javadoccomment"> * @param N Number of files.</em>
-<a class="jxr_linenumber" name="1421" href="#1421">1421</a> <em class="jxr_javadoccomment"> */</em>
-<a class="jxr_linenumber" name="1422" href="#1422">1422</a> <strong class="jxr_keyword">public</strong> <strong class="jxr_keyword">void</strong> compactRecentForTestingAssumingDefaultPolicy(<strong class="jxr_keyword">int</strong> N) <strong class="jxr_keyword">throws</strong> IOException {
-<a class="jxr_linenumber" name="1423" href="#1423">1423</a> List<StoreFile> filesToCompact;
-<a class="jxr_linenumber" name="1424" href="#1424">1424</a> <strong class="jxr_keyword">boolean</strong> isMajor;
-<a class="jxr_linenumber" name="1425" href="#1425">1425</a>
-<a class="jxr_linenumber" name="1426" href="#1426">1426</a> <strong class="jxr_keyword">this</strong>.lock.readLock().lock();
-<a class="jxr_linenumber" name="1427" href="#1427">1427</a> <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1428" href="#1428">1428</a> <strong class="jxr_keyword">synchronized</strong> (filesCompacting) {
-<a class="jxr_linenumber" name="1429" href="#1429">1429</a> filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
-<a class="jxr_linenumber" name="1430" href="#1430">1430</a> <strong class="jxr_keyword">if</strong> (!filesCompacting.isEmpty()) {
-<a class="jxr_linenumber" name="1431" href="#1431">1431</a> <em class="jxr_comment">// exclude all files older than the newest file we're currently</em>
-<a class="jxr_linenumber" name="1432" href="#1432">1432</a> <em class="jxr_comment">// compacting. this allows us to preserve contiguity (HBASE-2856)</em>
-<a class="jxr_linenumber" name="1433" href="#1433">1433</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> last = filesCompacting.get(filesCompacting.size() - 1);
-<a class="jxr_linenumber" name="1434" href="#1434">1434</a> <strong class="jxr_keyword">int</strong> idx = filesToCompact.indexOf(last);
-<a class="jxr_linenumber" name="1435" href="#1435">1435</a> Preconditions.checkArgument(idx != -1);
-<a class="jxr_linenumber" name="1436" href="#1436">1436</a> filesToCompact.subList(0, idx + 1).clear();
-<a class="jxr_linenumber" name="1437" href="#1437">1437</a> }
-<a class="jxr_linenumber" name="1438" href="#1438">1438</a> <strong class="jxr_keyword">int</strong> count = filesToCompact.size();
-<a class="jxr_linenumber" name="1439" href="#1439">1439</a> <strong class="jxr_keyword">if</strong> (N > count) {
-<a class="jxr_linenumber" name="1440" href="#1440">1440</a> <strong class="jxr_keyword">throw</strong> <strong class="jxr_keyword">new</strong> RuntimeException(<span class="jxr_string">"Not enough files"</span>);
-<a class="jxr_linenumber" name="1441" href="#1441">1441</a> }
-<a class="jxr_linenumber" name="1442" href="#1442">1442</a>
-<a class="jxr_linenumber" name="1443" href="#1443">1443</a> filesToCompact = filesToCompact.subList(count - N, count);
-<a class="jxr_linenumber" name="1444" href="#1444">1444</a> isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
-<a class="jxr_linenumber" name="1445" href="#1445">1445</a> filesCompacting.addAll(filesToCompact);
-<a class="jxr_linenumber" name="1446" href="#1446">1446</a> Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
-<a class="jxr_linenumber" name="1447" href="#1447">1447</a> }
-<a class="jxr_linenumber" name="1448" href="#1448">1448</a> } <strong class="jxr_keyword">finally</strong> {
-<a class="jxr_linenumber" name="1449" href="#1449">1449</a> <strong class="jxr_keyword">this</strong>.lock.readLock().unlock();
-<a class="jxr_linenumber" name="1450" href="#1450">1450</a> }
-<a class="jxr_linenumber" name="1451" href="#1451">1451</a>
-<a class="jxr_linenumber" name="1452" href="#1452">1452</a> <strong class="jxr_keyword">try</strong> {
-<a class="jxr_linenumber" name="1453" href="#1453">1453</a> <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
-<a class="jxr_linenumber" name="1454" href="#1454">1454</a> List<Path> newFiles = ((DefaultCompactor)<strong class="jxr_keyword">this</strong>.storeEngine.getCompactor())
-<a class="jxr_linenumber" name="1455" href="#1455">1455</a> .compactForTesting(filesToCompact, isMajor);
-<a class="jxr_linenumber" name="1456" href="#1456">1456</a> <strong class="jxr_keyword">for</strong> (Path newFile: newFiles) {
-<a class="jxr_linenumber" name="1457" href="#1457">1457</a> <em class="jxr_comment">// Move the compaction into place.</em>
-<a class="jxr_linenumber" name="1458" href="#1458">1458</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
-<a class="jxr_linenumber" name="1459" href="#1459">1459</a> <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
-<a class="jxr_linenumber" name="1460" href="#1460">1460</a> <strong class="jxr_keyword">this</strong>.getCoprocessorHost().postCompact(<strong class="jxr_keyword">this</strong>, sf, <strong class="jxr_keyword">null</strong>);
+<a class="jxr_linenumber" name="1211" href="#1211">1211</a>
+<a class="jxr_linenumber" name="1212" href="#1212">1212</a> <em class="jxr_comment">// Ready to go. Have list of files to compact.</em>
+<a class="jxr_linenumber" name="1213" href="#1213">1213</a> LOG.info(<span class="jxr_string">"Starting compaction of "</span> + filesToCompact.size() + <span class="jxr_string">" file(s) in "</span>
+<a class="jxr_linenumber" name="1214" href="#1214">1214</a> + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString()
+<a class="jxr_linenumber" name="1215" href="#1215">1215</a> + <span class="jxr_string">" into tmpdir="</span> + fs.getTempDir() + <span class="jxr_string">", totalSize="</span>
+<a class="jxr_linenumber" name="1216" href="#1216">1216</a> + TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(cr.getSize(), <span class="jxr_string">""</span>, 1));
+<a class="jxr_linenumber" name="1217" href="#1217">1217</a>
+<a class="jxr_linenumber" name="1218" href="#1218">1218</a> <em class="jxr_comment">// Commence the compaction.</em>
+<a class="jxr_linenumber" name="1219" href="#1219">1219</a> List<Path> newFiles = compaction.compact(throughputController, user);
+<a class="jxr_linenumber" name="1220" href="#1220">1220</a>
+<a class="jxr_linenumber" name="1221" href="#1221">1221</a> <em class="jxr_comment">// TODO: get rid of this!</em>
+<a class="jxr_linenumber" name="1222" href="#1222">1222</a> <strong class="jxr_keyword">if</strong> (!<strong class="jxr_keyword">this</strong>.conf.getBoolean(<span class="jxr_string">"hbase.hstore.compaction.complete"</span>, <strong class="jxr_keyword">true</strong>)) {
+<a class="jxr_linenumber" name="1223" href="#1223">1223</a> LOG.warn(<span class="jxr_string">"hbase.hstore.compaction.complete is set to false"</span>);
+<a class="jxr_linenumber" name="1224" href="#1224">1224</a> sfs = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(newFiles.size());
+<a class="jxr_linenumber" name="1225" href="#1225">1225</a> <strong class="jxr_keyword">final</strong> <strong class="jxr_keyword">boolean</strong> evictOnClose =
+<a class="jxr_linenumber" name="1226" href="#1226">1226</a> cacheConf != <strong class="jxr_keyword">null</strong>? cacheConf.shouldEvictOnClose(): <strong class="jxr_keyword">true</strong>;
+<a class="jxr_linenumber" name="1227" href="#1227">1227</a> <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
+<a class="jxr_linenumber" name="1228" href="#1228">1228</a> <em class="jxr_comment">// Create storefile around what we wrote with a reader on it.</em>
+<a class="jxr_linenumber" name="1229" href="#1229">1229</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = createStoreFileAndReader(newFile);
+<a class="jxr_linenumber" name="1230" href="#1230">1230</a> sf.closeReader(evictOnClose);
+<a class="jxr_linenumber" name="1231" href="#1231">1231</a> sfs.add(sf);
+<a class="jxr_linenumber" name="1232" href="#1232">1232</a> }
+<a class="jxr_linenumber" name="1233" href="#1233">1233</a> <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1234" href="#1234">1234</a> }
+<a class="jxr_linenumber" name="1235" href="#1235">1235</a> <em class="jxr_comment">// Do the steps necessary to complete the compaction.</em>
+<a class="jxr_linenumber" name="1236" href="#1236">1236</a> sfs = moveCompatedFilesIntoPlace(cr, newFiles, user);
+<a class="jxr_linenumber" name="1237" href="#1237">1237</a> writeCompactionWalRecord(filesToCompact, sfs);
+<a class="jxr_linenumber" name="1238" href="#1238">1238</a> replaceStoreFiles(filesToCompact, sfs);
+<a class="jxr_linenumber" name="1239" href="#1239">1239</a> <strong class="jxr_keyword">if</strong> (cr.isMajor()) {
+<a class="jxr_linenumber" name="1240" href="#1240">1240</a> majorCompactedCellsCount += getCompactionProgress().totalCompactingKVs;
+<a class="jxr_linenumber" name="1241" href="#1241">1241</a> majorCompactedCellsSize += getCompactionProgress().totalCompactedSize;
+<a class="jxr_linenumber" name="1242" href="#1242">1242</a> } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1243" href="#1243">1243</a> compactedCellsCount += getCompactionProgress().totalCompactingKVs;
+<a class="jxr_linenumber" name="1244" href="#1244">1244</a> compactedCellsSize += getCompactionProgress().totalCompactedSize;
+<a class="jxr_linenumber" name="1245" href="#1245">1245</a> }
+<a class="jxr_linenumber" name="1246" href="#1246">1246</a> <em class="jxr_comment">// At this point the store will use new files for all new scanners.</em>
+<a class="jxr_linenumber" name="1247" href="#1247">1247</a> completeCompaction(filesToCompact); <em class="jxr_comment">// update store size.</em>
+<a class="jxr_linenumber" name="1248" href="#1248">1248</a>
+<a class="jxr_linenumber" name="1249" href="#1249">1249</a> logCompactionEndMessage(cr, sfs, compactionStartTime);
+<a class="jxr_linenumber" name="1250" href="#1250">1250</a> <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1251" href="#1251">1251</a> } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1252" href="#1252">1252</a> finishCompactionRequest(cr);
+<a class="jxr_linenumber" name="1253" href="#1253">1253</a> }
+<a class="jxr_linenumber" name="1254" href="#1254">1254</a> }
+<a class="jxr_linenumber" name="1255" href="#1255">1255</a>
+<a class="jxr_linenumber" name="1256" href="#1256">1256</a> <strong class="jxr_keyword">private</strong> List<StoreFile> moveCompatedFilesIntoPlace(
+<a class="jxr_linenumber" name="1257" href="#1257">1257</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List<Path> newFiles, <a href="../../../../../org/apache/hadoop/hbase/security/User.html">User</a> user) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1258" href="#1258">1258</a> List<StoreFile> sfs = <strong class="jxr_keyword">new</strong> ArrayList<StoreFile>(newFiles.size());
+<a class="jxr_linenumber" name="1259" href="#1259">1259</a> <strong class="jxr_keyword">for</strong> (Path newFile : newFiles) {
+<a class="jxr_linenumber" name="1260" href="#1260">1260</a> assert newFile != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1261" href="#1261">1261</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> sf = moveFileIntoPlace(newFile);
+<a class="jxr_linenumber" name="1262" href="#1262">1262</a> <strong class="jxr_keyword">if</strong> (<strong class="jxr_keyword">this</strong>.getCoprocessorHost() != <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1263" href="#1263">1263</a> <strong class="jxr_keyword">final</strong> <a href="../../../../../org/apache/hadoop/hbase/regionserver/Store.html">Store</a> thisStore = <strong class="jxr_keyword">this</strong>;
+<a class="jxr_linenumber" name="1264" href="#1264">1264</a> <strong class="jxr_keyword">if</strong> (user == <strong class="jxr_keyword">null</strong>) {
+<a class="jxr_linenumber" name="1265" href="#1265">1265</a> getCoprocessorHost().postCompact(thisStore, sf, cr);
+<a class="jxr_linenumber" name="1266" href="#1266">1266</a> } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1267" href="#1267">1267</a> <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1268" href="#1268">1268</a> user.getUGI().doAs(<strong class="jxr_keyword">new</strong> PrivilegedExceptionAction<Void>() {
+<a class="jxr_linenumber" name="1269" href="#1269">1269</a> @Override
+<a class="jxr_linenumber" name="1270" href="#1270">1270</a> <strong class="jxr_keyword">public</strong> Void run() <strong class="jxr_keyword">throws</strong> Exception {
+<a class="jxr_linenumber" name="1271" href="#1271">1271</a> getCoprocessorHost().postCompact(thisStore, sf, cr);
+<a class="jxr_linenumber" name="1272" href="#1272">1272</a> <strong class="jxr_keyword">return</strong> <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1273" href="#1273">1273</a> }
+<a class="jxr_linenumber" name="1274" href="#1274">1274</a> });
+<a class="jxr_linenumber" name="1275" href="#1275">1275</a> } <strong class="jxr_keyword">catch</strong> (InterruptedException ie) {
+<a class="jxr_linenumber" name="1276" href="#1276">1276</a> InterruptedIOException iioe = <strong class="jxr_keyword">new</strong> InterruptedIOException();
+<a class="jxr_linenumber" name="1277" href="#1277">1277</a> iioe.initCause(ie);
+<a class="jxr_linenumber" name="1278" href="#1278">1278</a> <strong class="jxr_keyword">throw</strong> iioe;
+<a class="jxr_linenumber" name="1279" href="#1279">1279</a> }
+<a class="jxr_linenumber" name="1280" href="#1280">1280</a> }
+<a class="jxr_linenumber" name="1281" href="#1281">1281</a> }
+<a class="jxr_linenumber" name="1282" href="#1282">1282</a> assert sf != <strong class="jxr_keyword">null</strong>;
+<a class="jxr_linenumber" name="1283" href="#1283">1283</a> sfs.add(sf);
+<a class="jxr_linenumber" name="1284" href="#1284">1284</a> }
+<a class="jxr_linenumber" name="1285" href="#1285">1285</a> <strong class="jxr_keyword">return</strong> sfs;
+<a class="jxr_linenumber" name="1286" href="#1286">1286</a> }
+<a class="jxr_linenumber" name="1287" href="#1287">1287</a>
+<a class="jxr_linenumber" name="1288" href="#1288">1288</a> <em class="jxr_comment">// Package-visible for tests</em>
+<a class="jxr_linenumber" name="1289" href="#1289">1289</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/StoreFile.html">StoreFile</a> moveFileIntoPlace(<strong class="jxr_keyword">final</strong> Path newFile) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1290" href="#1290">1290</a> validateStoreFile(newFile);
+<a class="jxr_linenumber" name="1291" href="#1291">1291</a> <em class="jxr_comment">// Move the file into the right spot</em>
+<a class="jxr_linenumber" name="1292" href="#1292">1292</a> Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile);
+<a class="jxr_linenumber" name="1293" href="#1293">1293</a> <strong class="jxr_keyword">return</strong> createStoreFileAndReader(destPath);
+<a class="jxr_linenumber" name="1294" href="#1294">1294</a> }
+<a class="jxr_linenumber" name="1295" href="#1295">1295</a>
+<a class="jxr_linenumber" name="1296" href="#1296">1296</a> <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1297" href="#1297">1297</a> <em class="jxr_javadoccomment"> * Writes the compaction WAL record.</em>
+<a class="jxr_linenumber" name="1298" href="#1298">1298</a> <em class="jxr_javadoccomment"> * @param filesCompacted Files compacted (input).</em>
+<a class="jxr_linenumber" name="1299" href="#1299">1299</a> <em class="jxr_javadoccomment"> * @param newFiles Files from compaction.</em>
+<a class="jxr_linenumber" name="1300" href="#1300">1300</a> <em class="jxr_javadoccomment"> */</em>
+<a class="jxr_linenumber" name="1301" href="#1301">1301</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> writeCompactionWalRecord(Collection<StoreFile> filesCompacted,
+<a class="jxr_linenumber" name="1302" href="#1302">1302</a> Collection<StoreFile> newFiles) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1303" href="#1303">1303</a> <strong class="jxr_keyword">if</strong> (region.getWAL() == <strong class="jxr_keyword">null</strong>) <strong class="jxr_keyword">return</strong>;
+<a class="jxr_linenumber" name="1304" href="#1304">1304</a> List<Path> inputPaths = <strong class="jxr_keyword">new</strong> ArrayList<Path>(filesCompacted.size());
+<a class="jxr_linenumber" name="1305" href="#1305">1305</a> <strong class="jxr_keyword">for</strong> (StoreFile f : filesCompacted) {
+<a class="jxr_linenumber" name="1306" href="#1306">1306</a> inputPaths.add(f.getPath());
+<a class="jxr_linenumber" name="1307" href="#1307">1307</a> }
+<a class="jxr_linenumber" name="1308" href="#1308">1308</a> List<Path> outputPaths = <strong class="jxr_keyword">new</strong> ArrayList<Path>(newFiles.size());
+<a class="jxr_linenumber" name="1309" href="#1309">1309</a> <strong class="jxr_keyword">for</strong> (StoreFile f : newFiles) {
+<a class="jxr_linenumber" name="1310" href="#1310">1310</a> outputPaths.add(f.getPath());
+<a class="jxr_linenumber" name="1311" href="#1311">1311</a> }
+<a class="jxr_linenumber" name="1312" href="#1312">1312</a> <a href="../../../../../org/apache/hadoop/hbase/HRegionInfo.html">HRegionInfo</a> info = <strong class="jxr_keyword">this</strong>.region.getRegionInfo();
+<a class="jxr_linenumber" name="1313" href="#1313">1313</a> CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
+<a class="jxr_linenumber" name="1314" href="#1314">1314</a> family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
+<a class="jxr_linenumber" name="1315" href="#1315">1315</a> WALUtil.writeCompactionMarker(region.getWAL(), <strong class="jxr_keyword">this</strong>.region.getTableDesc(),
+<a class="jxr_linenumber" name="1316" href="#1316">1316</a> <strong class="jxr_keyword">this</strong>.region.getRegionInfo(), compactionDescriptor, region.getMVCC());
+<a class="jxr_linenumber" name="1317" href="#1317">1317</a> }
+<a class="jxr_linenumber" name="1318" href="#1318">1318</a>
+<a class="jxr_linenumber" name="1319" href="#1319">1319</a> @VisibleForTesting
+<a class="jxr_linenumber" name="1320" href="#1320">1320</a> <strong class="jxr_keyword">void</strong> replaceStoreFiles(<strong class="jxr_keyword">final</strong> Collection<StoreFile> compactedFiles,
+<a class="jxr_linenumber" name="1321" href="#1321">1321</a> <strong class="jxr_keyword">final</strong> Collection<StoreFile> result) <strong class="jxr_keyword">throws</strong> IOException {
+<a class="jxr_linenumber" name="1322" href="#1322">1322</a> <strong class="jxr_keyword">this</strong>.lock.writeLock().lock();
+<a class="jxr_linenumber" name="1323" href="#1323">1323</a> <strong class="jxr_keyword">try</strong> {
+<a class="jxr_linenumber" name="1324" href="#1324">1324</a> <strong class="jxr_keyword">this</strong>.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
+<a class="jxr_linenumber" name="1325" href="#1325">1325</a> filesCompacting.removeAll(compactedFiles); <em class="jxr_comment">// safe bc: lock.writeLock();</em>
+<a class="jxr_linenumber" name="1326" href="#1326">1326</a> } <strong class="jxr_keyword">finally</strong> {
+<a class="jxr_linenumber" name="1327" href="#1327">1327</a> <strong class="jxr_keyword">this</strong>.lock.writeLock().unlock();
+<a class="jxr_linenumber" name="1328" href="#1328">1328</a> }
+<a class="jxr_linenumber" name="1329" href="#1329">1329</a> }
+<a class="jxr_linenumber" name="1330" href="#1330">1330</a>
+<a class="jxr_linenumber" name="1331" href="#1331">1331</a> <em class="jxr_javadoccomment">/**</em>
+<a class="jxr_linenumber" name="1332" href="#1332">1332</a> <em class="jxr_javadoccomment"> * Log a very elaborate compaction completion message.</em>
+<a class="jxr_linenumber" name="1333" href="#1333">1333</a> <em class="jxr_javadoccomment"> * @param cr Request.</em>
+<a class="jxr_linenumber" name="1334" href="#1334">1334</a> <em class="jxr_javadoccomment"> * @param sfs Resulting files.</em>
+<a class="jxr_linenumber" name="1335" href="#1335">1335</a> <em class="jxr_javadoccomment"> * @param compactionStartTime Start time.</em>
+<a class="jxr_linenumber" name="1336" href="#1336">1336</a> <em class="jxr_javadoccomment"> */</em>
+<a class="jxr_linenumber" name="1337" href="#1337">1337</a> <strong class="jxr_keyword">private</strong> <strong class="jxr_keyword">void</strong> logCompactionEndMessage(
+<a class="jxr_linenumber" name="1338" href="#1338">1338</a> <a href="../../../../../org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.html">CompactionRequest</a> cr, List<StoreFile> sfs, <strong class="jxr_keyword">long</strong> compactionStartTime) {
+<a class="jxr_linenumber" name="1339" href="#1339">1339</a> <strong class="jxr_keyword">long</strong> now = EnvironmentEdgeManager.currentTime();
+<a class="jxr_linenumber" name="1340" href="#1340">1340</a> StringBuilder message = <strong class="jxr_keyword">new</strong> StringBuilder(
+<a class="jxr_linenumber" name="1341" href="#1341">1341</a> <span class="jxr_string">"Completed"</span> + (cr.isMajor() ? <span class="jxr_string">" major"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" compaction of "</span>
+<a class="jxr_linenumber" name="1342" href="#1342">1342</a> + cr.getFiles().size() + (cr.isAllFiles() ? <span class="jxr_string">" (all)"</span> : <span class="jxr_string">""</span>) + <span class="jxr_string">" file(s) in "</span>
+<a class="jxr_linenumber" name="1343" href="#1343">1343</a> + <strong class="jxr_keyword">this</strong> + <span class="jxr_string">" of "</span> + <strong class="jxr_keyword">this</strong>.getRegionInfo().getRegionNameAsString() + <span class="jxr_string">" into "</span>);
+<a class="jxr_linenumber" name="1344" href="#1344">1344</a> <strong class="jxr_keyword">if</strong> (sfs.isEmpty()) {
+<a class="jxr_linenumber" name="1345" href="#1345">1345</a> message.append(<span class="jxr_string">"none, "</span>);
+<a class="jxr_linenumber" name="1346" href="#1346">1346</a> } <strong class="jxr_keyword">else</strong> {
+<a class="jxr_linenumber" name="1347" href="#1347">1347</a> <strong class="jxr_keyword">for</strong> (StoreFile sf: sfs) {
+<a class="jxr_linenumber" name="1348" href="#1348">1348</a> message.append(sf.getPath().getName());
+<a class="jxr_linenumber" name="1349" href="#1349">1349</a> message.append(<span class="jxr_string">"(size="</span>);
+<a class="jxr_linenumber" name="1350" href="#1350">1350</a> message.append(TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(sf.getReader().length(), <span class="jxr_string">""</span>, 1));
+<a class="jxr_linenumber" name="1351" href="#1351">1351</a> message.append(<span class="jxr_string">"), "</span>);
+<a class="jxr_linenumber" name="1352" href="#1352">1352</a> }
+<a class="jxr_linenumber" name="1353" href="#1353">1353</a> }
+<a class="jxr_linenumber" name="1354" href="#1354">1354</a> message.append(<span class="jxr_string">"total size for store is "</span>)
+<a class="jxr_linenumber" name="1355" href="#1355">1355</a> .append(StringUtils.TraditionalBinaryPrefix.<strong class="jxr_keyword">long</strong>2String(storeSize, <span class="jxr_string">""</span>, 1))
+<a class="jxr_linenumber" name="1356" href="#1356">1356</a> .append(<span class="jxr_string">". This selection was in queue for "</span>)
+<a class="jxr_linenumber" name="1357" href="#1357">1357</a> .append(StringUtils.formatTimeDiff(compactionStartTime, cr.getSelectionTime()))
+<a class="jxr_linenumber" name="1358" href="#1358">1358</a> .append(<span class="jxr_string">", and took "</span>).append(StringUtils.formatTimeDiff(now, compactionStartTime))
+<a class="jxr_linenumber" name="1359" href="#1359">1359</a> .append(<span class="jxr_string">" to execute."</span>);
+<a class="jxr_linenumber" name="1360" href="#1360">1360</a> LOG.info(message.toString());
+<a class="jxr_linenumber" name="1361" href="#1361">1361</a> <strong class="jxr_keyword">if</strong> (LOG.isTraceEnabled()) {
+<a class="jxr_linenumber" name="1362" href="#1362">1362</a> <strong class="jxr_keyword">int</strong> fileCount = storeEngine.getStoreFileManager().getStorefileCount();
+<a class="jxr_linenumber" name="1363" href="#1363">1363</a> <strong class="jxr_keyword">long</strong> resultSize = 0;
+<a class="jxr_linenumber" name="1364" href="#1364">1364</a> <strong class="jxr_keyword">for</strong> (StoreFile sf : sfs) {
+<a class="jxr_linenumber" name="1365" href="#1365">1365</a> resultSize += sf.getReader().length();
+<a class="jxr_linenumber" name="1366" href="#1366">1366</a> }
+<a class="jxr_linenumber" name="1367" href="#1367">1367</a>
<TRUNCATED>