You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2017/08/25 02:17:48 UTC

[1/2] kudu git commit: KUDU-1943: Add BlockTransaction to Block Manager

Repository: kudu
Updated Branches:
  refs/heads/master 81fad4da9 -> 3d42a264c


KUDU-1943: Add BlockTransaction to Block Manager

This adds a new layer of abstraction 'BlockTransaction' to
Block Manager, to accumulate new blocks in a logical operation,
e.g. flush/compaction. Avoid using fsync() per block can
potentionaly reduce a lot I/O overhead.

This patch also add a new API, Finalize(), to Block Manager,
so that an LBM container can be reused without flushing
in-flight writable blocks to disk. This can bring down the
log block container number and disk space consumption for
first few flush/compaction operations when using log block
manager.

I performed manual test which creates a table with two tablets,
'kudu perf loadgen localhost --num-rows-per-thread=20000000
 --num-threads=10 --keep-auto-table --table_num_buckets=2'.
With this patch, 'log_block_manager_containers' dropped from 56
to 7.

A design doc can be found here:
https://docs.google.com/document/d/1cB8pDs-Ho7p2HlzkWo1d-wmXr90YntIjdvdOv7u7gn0/edit?usp=sharing

Change-Id: I01f3ed830cc1f86d7bb8a66b2f780914a88884bd
Reviewed-on: http://gerrit.cloudera.org:8080/7207
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/59ca360b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/59ca360b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/59ca360b

Branch: refs/heads/master
Commit: 59ca360bff394fa18da32487c44ec876feea9510
Parents: 81fad4d
Author: hahao <ha...@cloudera.com>
Authored: Wed Aug 23 20:21:06 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Aug 25 02:16:14 2017 +0000

----------------------------------------------------------------------
 src/kudu/cfile/bloomfile.cc              |  12 +-
 src/kudu/cfile/bloomfile.h               |   7 +-
 src/kudu/cfile/cfile-test.cc             |  26 +-
 src/kudu/cfile/cfile_writer.cc           |  42 +--
 src/kudu/cfile/cfile_writer.h            |   5 +-
 src/kudu/fs/block_manager-stress-test.cc |  94 +++---
 src/kudu/fs/block_manager-test.cc        | 139 +++++++--
 src/kudu/fs/block_manager.cc             |  23 +-
 src/kudu/fs/block_manager.h              |  87 +++---
 src/kudu/fs/file_block_manager.cc        |  47 ++-
 src/kudu/fs/file_block_manager.h         |   2 +-
 src/kudu/fs/log_block_manager-test.cc    |  69 ++++-
 src/kudu/fs/log_block_manager.cc         | 425 ++++++++++++++------------
 src/kudu/fs/log_block_manager.h          |   9 +-
 src/kudu/tablet/deltafile.cc             |  12 +-
 src/kudu/tablet/deltafile.h              |   8 +-
 src/kudu/tablet/diskrowset.cc            |  25 +-
 src/kudu/tablet/diskrowset.h             |  12 +-
 src/kudu/tablet/multi_column_writer.cc   |  12 +-
 src/kudu/tablet/multi_column_writer.h    |   8 +-
 20 files changed, 613 insertions(+), 451 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/cfile/bloomfile.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/bloomfile.cc b/src/kudu/cfile/bloomfile.cc
index 520a743..f5c793b 100644
--- a/src/kudu/cfile/bloomfile.cc
+++ b/src/kudu/cfile/bloomfile.cc
@@ -57,8 +57,8 @@ using std::vector;
 namespace kudu {
 namespace cfile {
 
+using fs::BlockTransaction;
 using fs::ReadableBlock;
-using fs::ScopedWritableBlockCloser;
 using fs::WritableBlock;
 
 // Generator for BloomFileReader::instance_nonce_.
@@ -113,16 +113,16 @@ Status BloomFileWriter::Start() {
 }
 
 Status BloomFileWriter::Finish() {
-  ScopedWritableBlockCloser closer;
-  RETURN_NOT_OK(FinishAndReleaseBlock(&closer));
-  return closer.CloseBlocks();
+  BlockTransaction transaction;
+  RETURN_NOT_OK(FinishAndReleaseBlock(&transaction));
+  return transaction.CommitCreatedBlocks();
 }
 
-Status BloomFileWriter::FinishAndReleaseBlock(ScopedWritableBlockCloser* closer) {
+Status BloomFileWriter::FinishAndReleaseBlock(BlockTransaction* transaction) {
   if (bloom_builder_.count() > 0) {
     RETURN_NOT_OK(FinishCurrentBloomBlock());
   }
-  return writer_->FinishAndReleaseBlock(closer);
+  return writer_->FinishAndReleaseBlock(transaction);
 }
 
 size_t BloomFileWriter::written_size() const {

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/cfile/bloomfile.h
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/bloomfile.h b/src/kudu/cfile/bloomfile.h
index a7a1f8c..bf59705 100644
--- a/src/kudu/cfile/bloomfile.h
+++ b/src/kudu/cfile/bloomfile.h
@@ -35,8 +35,8 @@
 namespace kudu {
 
 namespace fs {
+class BlockTransaction;
 class ReadableBlock;
-class ScopedWritableBlockCloser;
 class WritableBlock;
 }
 
@@ -56,8 +56,9 @@ class BloomFileWriter {
   // Close the bloom's CFile, closing the underlying writable block.
   Status Finish();
 
-  // Close the bloom's CFile, releasing the underlying block to 'closer'.
-  Status FinishAndReleaseBlock(fs::ScopedWritableBlockCloser* closer);
+  // Close the bloom's CFile, finalizing the underlying block and
+  // releasing it to 'transaction'.
+  Status FinishAndReleaseBlock(fs::BlockTransaction* transaction);
 
   // Estimate the amount of data already written to this file.
   size_t written_size() const;

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/cfile/cfile-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile-test.cc b/src/kudu/cfile/cfile-test.cc
index 8525d4d..3d55e93 100644
--- a/src/kudu/cfile/cfile-test.cc
+++ b/src/kudu/cfile/cfile-test.cc
@@ -22,7 +22,7 @@
 #include <ctime>
 #include <functional>
 #include <memory>
-#include <ostream>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -73,8 +73,6 @@
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
-DECLARE_string(block_cache_type);
-DECLARE_string(cfile_do_on_finish);
 DECLARE_bool(cfile_write_checksums);
 DECLARE_bool(cfile_verify_checksums);
 
@@ -936,22 +934,12 @@ TEST_P(TestCFileBothCacheTypes, TestReleaseBlock) {
   WriterOptions opts;
   CFileWriter w(opts, GetTypeInfo(STRING), false, std::move(sink));
   ASSERT_OK(w.Start());
-  fs::ScopedWritableBlockCloser closer;
-  ASSERT_OK(w.FinishAndReleaseBlock(&closer));
-  if (FLAGS_cfile_do_on_finish == "flush") {
-    ASSERT_EQ(1, closer.blocks().size());
-    ASSERT_EQ(WritableBlock::FLUSHING, closer.blocks()[0]->state());
-  } else if (FLAGS_cfile_do_on_finish == "close") {
-    ASSERT_EQ(0, closer.blocks().size());
-  } else if (FLAGS_cfile_do_on_finish == "nothing") {
-    ASSERT_EQ(1, closer.blocks().size());
-    ASSERT_EQ(WritableBlock::DIRTY, closer.blocks()[0]->state());
-  } else {
-    LOG(FATAL) << "Unknown value for cfile_do_on_finish: "
-               << FLAGS_cfile_do_on_finish;
-  }
-  ASSERT_OK(closer.CloseBlocks());
-  ASSERT_EQ(0, closer.blocks().size());
+  fs::BlockTransaction transaction;
+  ASSERT_OK(w.FinishAndReleaseBlock(&transaction));
+  ASSERT_EQ(1, transaction.created_blocks().size());
+  ASSERT_EQ(WritableBlock::FINALIZED, transaction.created_blocks()[0]->state());
+  ASSERT_OK(transaction.CommitCreatedBlocks());
+  ASSERT_EQ(0, transaction.created_blocks().size());
 }
 
 TEST_P(TestCFileBothCacheTypes, TestLazyInit) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/cfile/cfile_writer.cc
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile_writer.cc b/src/kudu/cfile/cfile_writer.cc
index 9ddcbfb..4506749 100644
--- a/src/kudu/cfile/cfile_writer.cc
+++ b/src/kudu/cfile/cfile_writer.cc
@@ -18,7 +18,6 @@
 #include "kudu/cfile/cfile_writer.h"
 
 #include <numeric>
-#include <string>
 #include <ostream>
 #include <utility>
 
@@ -54,30 +53,12 @@ DEFINE_string(cfile_default_compression_codec, "none",
               "Default cfile block compression codec.");
 TAG_FLAG(cfile_default_compression_codec, advanced);
 
-// The default value is optimized for throughput in the case that
-// there are multiple drives backing the tablet. By asynchronously
-// flushing each cfile before issuing any fsyncs, the IO across
-// disks is done in parallel.
-//
-// This increases throughput but can harm latency in the case that
-// there are few disks and the WAL is on the same disk as the
-// data blocks. The default is chosen based on the assumptions that:
-// - latency is leveled across machines by Raft
-// - latency-sensitive applications can devote a disk to the WAL
-// - super-sensitive applications can devote an SSD to the WAL.
-// - users could always change this to "close", which slows down throughput
-//   but may improve write latency.
-DEFINE_string(cfile_do_on_finish, "flush",
-              "What to do to cfile blocks when writing is finished. "
-              "Possible values are 'close', 'flush', or 'nothing'.");
-TAG_FLAG(cfile_do_on_finish, experimental);
-
 DEFINE_bool(cfile_write_checksums, true,
             "Write CRC32 checksums for each block");
 TAG_FLAG(cfile_write_checksums, evolving);
 
 using google::protobuf::RepeatedPtrField;
-using kudu::fs::ScopedWritableBlockCloser;
+using kudu::fs::BlockTransaction;
 using kudu::fs::WritableBlock;
 using std::accumulate;
 using std::pair;
@@ -221,12 +202,12 @@ Status CFileWriter::Start() {
 
 Status CFileWriter::Finish() {
   TRACE_EVENT0("cfile", "CFileWriter::Finish");
-  ScopedWritableBlockCloser closer;
-  RETURN_NOT_OK(FinishAndReleaseBlock(&closer));
-  return closer.CloseBlocks();
+  BlockTransaction transaction;
+  RETURN_NOT_OK(FinishAndReleaseBlock(&transaction));
+  return transaction.CommitCreatedBlocks();
 }
 
-Status CFileWriter::FinishAndReleaseBlock(ScopedWritableBlockCloser* closer) {
+Status CFileWriter::FinishAndReleaseBlock(BlockTransaction* transaction) {
   TRACE_EVENT0("cfile", "CFileWriter::FinishAndReleaseBlock");
   CHECK(state_ == kWriterWriting) <<
     "Bad state for Finish(): " << state_;
@@ -290,17 +271,8 @@ Status CFileWriter::FinishAndReleaseBlock(ScopedWritableBlockCloser* closer) {
   RETURN_NOT_OK_PREPEND(WriteRawData(footer_slices), "Couldn't write footer");
 
   // Done with this block.
-  if (FLAGS_cfile_do_on_finish == "flush") {
-    RETURN_NOT_OK(block_->FlushDataAsync());
-    closer->AddBlock(std::move(block_));
-  } else if (FLAGS_cfile_do_on_finish == "close") {
-    RETURN_NOT_OK(block_->Close());
-  } else if (FLAGS_cfile_do_on_finish == "nothing") {
-    closer->AddBlock(std::move(block_));
-  } else {
-    LOG(FATAL) << "Unknown value for cfile_do_on_finish: "
-               << FLAGS_cfile_do_on_finish;
-  }
+  RETURN_NOT_OK(block_->Finalize());
+  transaction->AddCreatedBlock(std::move(block_));
   return Status::OK();
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/cfile/cfile_writer.h
----------------------------------------------------------------------
diff --git a/src/kudu/cfile/cfile_writer.h b/src/kudu/cfile/cfile_writer.h
index b3461b2..4800573 100644
--- a/src/kudu/cfile/cfile_writer.h
+++ b/src/kudu/cfile/cfile_writer.h
@@ -114,8 +114,9 @@ class CFileWriter {
   // Close the CFile and close the underlying writable block.
   Status Finish();
 
-  // Close the CFile and release the underlying writable block to 'closer'.
-  Status FinishAndReleaseBlock(fs::ScopedWritableBlockCloser* closer);
+  // Close the CFile, finalizing the underlying block and releasing
+  // it to 'transaction'.
+  Status FinishAndReleaseBlock(fs::BlockTransaction* transaction);
 
   bool finished() {
     return state_ == kWriterFinished;

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/block_manager-stress-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager-stress-test.cc b/src/kudu/fs/block_manager-stress-test.cc
index e03cbc4..e8d280b 100644
--- a/src/kudu/fs/block_manager-stress-test.cc
+++ b/src/kudu/fs/block_manager-stress-test.cc
@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
 #include <cmath>
 #include <cstdint>
 #include <cstring>
@@ -43,7 +44,6 @@
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/ref_counted.h"
-#include "kudu/gutil/stl_util.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/atomic.h"
@@ -76,6 +76,7 @@ DEFINE_int32(minimum_live_blocks_for_delete, 1000,
              "threads will not delete any");
 DEFINE_int32(block_group_size, 8, "Number of blocks to write per block "
              "group. Must be power of 2");
+DEFINE_int32(block_group_number, 2, "Total number of block groups.");
 DEFINE_int32(block_group_bytes, 32 * 1024,
              "Total amount of data (in bytes) to write per block group");
 DEFINE_int32(num_bytes_per_write, 32,
@@ -302,59 +303,67 @@ void BlockManagerStressTest<T>::WriterThread() {
   size_t num_bytes_written = 0;
   MonoDelta tight_loop(MonoDelta::FromSeconds(0));
   while (!ShouldStop(tight_loop)) {
-    vector<WritableBlock*> dirty_blocks;
-    ElementDeleter deleter(&dirty_blocks);
-    vector<Random> dirty_block_rands;
+    vector<unique_ptr<WritableBlock>> all_dirty_blocks;
+    for (int i = 0; i < FLAGS_block_group_number; i++) {
+      vector<unique_ptr<WritableBlock>> dirty_blocks;
+      vector<Random> dirty_block_rands;
+
+      // Create the blocks and write out the PRNG seeds.
+      for (int j = 0; j < FLAGS_block_group_size; j++) {
+        unique_ptr<WritableBlock> block;
+        CHECK_OK(bm_->CreateBlock(CreateBlockOptions({test_tablet_name_}), &block));
+
+        const uint32_t seed = rand.Next() + 1;
+        Slice seed_slice(reinterpret_cast<const uint8_t*>(&seed), sizeof(seed));
+        CHECK_OK(block->Append(seed_slice));
+
+        dirty_blocks.emplace_back(std::move(block));
+        dirty_block_rands.emplace_back(seed);
+      }
 
-    // Create the blocks and write out the PRNG seeds.
-    for (int i = 0; i < FLAGS_block_group_size; i++) {
-      unique_ptr<WritableBlock> block;
-      CHECK_OK(bm_->CreateBlock(CreateBlockOptions({ test_tablet_name_ }), &block));
+      // Write a large amount of data to the group of blocks.
+      //
+      // To emulate a real life workload, we pick the next block to write at
+      // random, and write a smaller chunk of data to it.
+      size_t total_dirty_bytes = 0;
+      while (total_dirty_bytes < FLAGS_block_group_bytes) {
+        // Pick the next block.
+        int next_block_idx = rand.Skewed(log2(dirty_blocks.size()));
+        WritableBlock* block = dirty_blocks[next_block_idx].get();
+        Random& rand = dirty_block_rands[next_block_idx];
+
+        // Write a small chunk of data.
+        faststring data;
+        while (data.length() < FLAGS_num_bytes_per_write) {
+          const uint32_t next_int = rand.Next();
+          data.append(&next_int, sizeof(next_int));
+        }
+        CHECK_OK(block->Append(data));
+        total_dirty_bytes += data.length();
+      }
 
-      const uint32_t seed = rand.Next() + 1;
-      Slice seed_slice(reinterpret_cast<const uint8_t*>(&seed), sizeof(seed));
-      CHECK_OK(block->Append(seed_slice));
+      // Finalize the dirty blocks.
+      for (const auto& dirty_block : dirty_blocks) {
+        CHECK_OK(dirty_block->Finalize());
+      }
 
-      dirty_blocks.push_back(block.release());
-      dirty_block_rands.emplace_back(seed);
-    }
+      num_blocks_written += dirty_blocks.size();
+      num_bytes_written += total_dirty_bytes;
 
-    // Write a large amount of data to the group of blocks.
-    //
-    // To emulate a real life workload, we pick the next block to write at
-    // random, and write a smaller chunk of data to it.
-    size_t total_dirty_bytes = 0;
-    while (total_dirty_bytes < FLAGS_block_group_bytes) {
-      // Pick the next block.
-      int next_block_idx = rand.Skewed(log2(dirty_blocks.size()));
-      WritableBlock* block = dirty_blocks[next_block_idx];
-      Random& rand = dirty_block_rands[next_block_idx];
-
-      // Write a small chunk of data.
-      faststring data;
-      while (data.length() < FLAGS_num_bytes_per_write) {
-        const uint32_t next_int = rand.Next();
-        data.append(&next_int, sizeof(next_int));
+      for (auto& dirty_block : dirty_blocks) {
+        all_dirty_blocks.emplace_back(std::move(dirty_block));
       }
-      CHECK_OK(block->Append(data));
-      total_dirty_bytes += data.length();
     }
 
     // Close all dirty blocks.
-    //
-    // We could close them implicitly when the blocks are destructed but
-    // this way we can check for errors.
-    CHECK_OK(bm_->CloseBlocks(dirty_blocks));
-
+    CHECK_OK(bm_->CloseBlocks(all_dirty_blocks));
     // Publish the now sync'ed blocks to readers and deleters.
     {
       std::lock_guard<simple_spinlock> l(lock_);
-      for (WritableBlock* block : dirty_blocks) {
+      for (const auto& block : all_dirty_blocks) {
         InsertOrDie(&written_blocks_, block->id(), 0);
       }
     }
-    num_blocks_written += dirty_blocks.size();
-    num_bytes_written += total_dirty_bytes;
   }
 
   total_blocks_written_.IncrementBy(num_blocks_written);
@@ -471,7 +480,7 @@ template <>
 int BlockManagerStressTest<FileBlockManager>::GetMaxFdCount() const {
   return FLAGS_block_manager_max_open_files +
       // Each open block exists outside the file cache.
-      (FLAGS_num_writer_threads * FLAGS_block_group_size) +
+      (FLAGS_num_writer_threads * FLAGS_block_group_size * FLAGS_block_group_number) +
       // Each reader thread can open a file outside the cache if its lookup
       // misses. It'll immediately evict an existing fd, but in that brief
       // window of time both fds may be open simultaneously.
@@ -484,7 +493,7 @@ int BlockManagerStressTest<LogBlockManager>::GetMaxFdCount() const {
       // If all containers are full, each open block could theoretically
       // result in a new container, which is two files briefly outside the
       // cache (before they are inserted and evict other cached files).
-      (FLAGS_num_writer_threads * FLAGS_block_group_size * 2);
+      (FLAGS_num_writer_threads * FLAGS_block_group_size * FLAGS_block_group_number * 2);
 }
 
 template <>
@@ -513,6 +522,7 @@ TYPED_TEST_CASE(BlockManagerStressTest, BlockManagers);
 TYPED_TEST(BlockManagerStressTest, StressTest) {
   OverrideFlagForSlowTests("test_duration_secs", "30");
   OverrideFlagForSlowTests("block_group_size", "16");
+  OverrideFlagForSlowTests("block_group_number", "10");
   OverrideFlagForSlowTests("num_inconsistencies", "128");
 
   const int kNumStarts = 3;

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager-test.cc b/src/kudu/fs/block_manager-test.cc
index 8755d46..91fd210 100644
--- a/src/kudu/fs/block_manager-test.cc
+++ b/src/kudu/fs/block_manager-test.cc
@@ -20,7 +20,9 @@
 #include <cstdint>
 #include <memory>
 #include <ostream>
+#include <stdlib.h>
 #include <string>
+#include <thread>
 #include <unordered_set>
 #include <vector>
 
@@ -43,11 +45,11 @@
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/ref_counted.h"
-#include "kudu/gutil/stl_util.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/env.h"
 #include "kudu/util/mem_tracker.h"
 #include "kudu/util/metrics.h"
+#include "kudu/util/monotime.h"
 #include "kudu/util/path_util.h"
 #include "kudu/util/pb_util.h"
 #include "kudu/util/random.h"
@@ -129,14 +131,14 @@ class BlockManagerTest : public KuduTest {
     int num_blocks = num_dirs * num_blocks_per_dir;
 
     // Write 'num_blocks' blocks to this data dir group.
-    ScopedWritableBlockCloser closer;
+    BlockTransaction transaction;
     for (int i = 0; i < num_blocks; i++) {
       unique_ptr<WritableBlock> written_block;
       ASSERT_OK(bm_->CreateBlock(opts, &written_block));
       ASSERT_OK(written_block->Append(kTestData));
-      closer.AddBlock(std::move(written_block));
+      transaction.AddCreatedBlock(std::move(written_block));
     }
-    ASSERT_OK(closer.CloseBlocks());
+    ASSERT_OK(transaction.CommitCreatedBlocks());
   }
 
  protected:
@@ -346,15 +348,15 @@ void BlockManagerTest<LogBlockManager>::RunMultipathTest(const vector<string>& p
   ASSERT_OK(dd_manager_->CreateDataDirGroup("multipath_test"));
 
   const char* kTestData = "test data";
-  ScopedWritableBlockCloser closer;
+  BlockTransaction transaction;
   // Creates (numPaths * 2) containers.
   for (int j = 0; j < paths.size() * 2; j++) {
     unique_ptr<WritableBlock> block;
     ASSERT_OK(bm_->CreateBlock(opts, &block));
     ASSERT_OK(block->Append(kTestData));
-    closer.AddBlock(std::move(block));
+    transaction.AddCreatedBlock(std::move(block));
   }
-  ASSERT_OK(closer.CloseBlocks());
+  ASSERT_OK(transaction.CommitCreatedBlocks());
 
   // Verify the results. (numPaths * 2) containers were created, each
   // consisting of 2 files. Thus, there should be a total of
@@ -525,12 +527,8 @@ TYPED_TEST(BlockManagerTest, CloseManyBlocksTest) {
     return;
   }
 
-  // Disable preallocation for this test as it creates many containers.
-  FLAGS_log_container_preallocate_bytes = 0;
-
   Random rand(SeedRandom());
-  vector<WritableBlock*> dirty_blocks;
-  ElementDeleter deleter(&dirty_blocks);
+  vector<unique_ptr<WritableBlock>> dirty_blocks;
   LOG_TIMING(INFO, Substitute("creating $0 blocks", kNumBlocks)) {
     for (int i = 0; i < kNumBlocks; i++) {
       // Create a block.
@@ -543,8 +541,8 @@ TYPED_TEST(BlockManagerTest, CloseManyBlocksTest) {
         data[i] = rand.Next();
       }
       written_block->Append(Slice(data, sizeof(data)));
-
-      dirty_blocks.push_back(written_block.release());
+      written_block->Finalize();
+      dirty_blocks.emplace_back(std::move(written_block));
     }
   }
 
@@ -553,16 +551,6 @@ TYPED_TEST(BlockManagerTest, CloseManyBlocksTest) {
   }
 }
 
-// We can't really test that FlushDataAsync() "works", but we can test that
-// it doesn't break anything.
-TYPED_TEST(BlockManagerTest, FlushDataAsyncTest) {
-  unique_ptr<WritableBlock> written_block;
-  ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
-  string test_data = "test data";
-  ASSERT_OK(written_block->Append(test_data));
-  ASSERT_OK(written_block->FlushDataAsync());
-}
-
 TYPED_TEST(BlockManagerTest, WritableBlockStateTest) {
   unique_ptr<WritableBlock> written_block;
 
@@ -577,11 +565,11 @@ TYPED_TEST(BlockManagerTest, WritableBlockStateTest) {
   ASSERT_OK(written_block->Close());
   ASSERT_EQ(WritableBlock::CLOSED, written_block->state());
 
-  // Test FLUSHING->CLOSED transition.
+  // Test FINALIZED->CLOSED transition.
   ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
   ASSERT_OK(written_block->Append(test_data));
-  ASSERT_OK(written_block->FlushDataAsync());
-  ASSERT_EQ(WritableBlock::FLUSHING, written_block->state());
+  ASSERT_OK(written_block->Finalize());
+  ASSERT_EQ(WritableBlock::FINALIZED, written_block->state());
   ASSERT_OK(written_block->Close());
   ASSERT_EQ(WritableBlock::CLOSED, written_block->state());
 
@@ -590,10 +578,10 @@ TYPED_TEST(BlockManagerTest, WritableBlockStateTest) {
   ASSERT_OK(written_block->Close());
   ASSERT_EQ(WritableBlock::CLOSED, written_block->state());
 
-  // Test FlushDataAsync() no-op.
+  // Test Finalize() no-op.
   ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
-  ASSERT_OK(written_block->FlushDataAsync());
-  ASSERT_EQ(WritableBlock::FLUSHING, written_block->state());
+  ASSERT_OK(written_block->Finalize());
+  ASSERT_EQ(WritableBlock::FINALIZED, written_block->state());
 
   // Test DIRTY->CLOSED transition.
   ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
@@ -639,7 +627,7 @@ TYPED_TEST(BlockManagerTest, AbortTest) {
 
   ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
   ASSERT_OK(written_block->Append(test_data));
-  ASSERT_OK(written_block->FlushDataAsync());
+  ASSERT_OK(written_block->Finalize());
   ASSERT_OK(written_block->Abort());
   ASSERT_EQ(WritableBlock::CLOSED, written_block->state());
   ASSERT_TRUE(this->bm_->OpenBlock(written_block->id(), nullptr)
@@ -898,7 +886,7 @@ TYPED_TEST(BlockManagerTest, TestMetadataOkayDespiteFailure) {
       RETURN_NOT_OK(block->Append(test_data));
     }
 
-    RETURN_NOT_OK(block->FlushDataAsync());
+    RETURN_NOT_OK(block->Finalize());
     RETURN_NOT_OK(block->Close());
     *out = block->id();
     return Status::OK();
@@ -1017,5 +1005,92 @@ TYPED_TEST(BlockManagerTest, TestGetAllBlockIds) {
   ASSERT_EQ(ids, retrieved_ids);
 }
 
+TYPED_TEST(BlockManagerTest, FinalizeTest) {
+  const string kTestData = "test data";
+  unique_ptr<WritableBlock> written_block;
+  ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &written_block));
+  ASSERT_OK(written_block->Append(kTestData));
+  ASSERT_OK(written_block->Finalize());
+  ASSERT_OK(written_block->Close());
+}
+
+// Tests that WritableBlock::Close() is thread-safe for multiple
+// FINALIZED blocks.
+TYPED_TEST(BlockManagerTest, ConcurrentCloseFinalizedWritableBlockTest) {
+  const string kTestData = "test data";
+
+  // Create several blocks. For each block, Finalize() it after fully
+  // written, and then Close() it.
+  auto write_data = [&]() {
+    for (int i = 0; i < 5; i++) {
+      unique_ptr<WritableBlock> writer;
+      CHECK_OK(this->bm_->CreateBlock(this->test_block_opts_, &writer));
+      CHECK_OK(writer->Append(kTestData));
+      CHECK_OK(writer->Finalize());
+      SleepFor(MonoDelta::FromMilliseconds(rand() % 100));
+      CHECK_OK(writer->Close());
+    }
+  };
+
+  vector<std::thread> threads;
+  for (int i = 0; i < 100; i++) {
+    threads.emplace_back(write_data);
+  }
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  vector<BlockId> retrieved_ids;
+  ASSERT_OK(this->bm_->GetAllBlockIds(&retrieved_ids));
+  ASSERT_EQ(500, retrieved_ids.size());
+  for (const auto& id : retrieved_ids) {
+    unique_ptr<ReadableBlock> block;
+    ASSERT_OK(this->bm_->OpenBlock(id, &block));
+    uint64_t size;
+    ASSERT_OK(block->Size(&size));
+    uint8_t buf[size];
+    Slice slice(buf, size);
+    ASSERT_OK(block->Read(0, &slice));
+    ASSERT_EQ(kTestData, slice);
+  }
+
+  ASSERT_OK(this->ReopenBlockManager(scoped_refptr<MetricEntity>(),
+                                     shared_ptr<MemTracker>(),
+                                     { GetTestDataDirectory() },
+                                     false /* create */));
+}
+
+TYPED_TEST(BlockManagerTest, TestBlockTransaction) {
+  // Create a BlockTransaction. In this transaction,
+  // create some blocks and commit the writes all together.
+  const string kTestData = "test data";
+  BlockTransaction transaction;
+  vector<BlockId> block_ids;
+  for (int i = 0; i < 20; i++) {
+    unique_ptr<WritableBlock> writer;
+    ASSERT_OK(this->bm_->CreateBlock(this->test_block_opts_, &writer));
+
+    // Write some data to it.
+    ASSERT_OK(writer->Append(kTestData));
+    ASSERT_OK(writer->Finalize());
+    block_ids.emplace_back(writer->id());
+    transaction.AddCreatedBlock(std::move(writer));
+  }
+  ASSERT_OK(transaction.CommitCreatedBlocks());
+
+  // Read the blocks and verify the content.
+  for (const auto& block : block_ids) {
+    unique_ptr<ReadableBlock> reader;
+    ASSERT_OK(this->bm_->OpenBlock(block, &reader));
+    uint64_t sz;
+    ASSERT_OK(reader->Size(&sz));
+    ASSERT_EQ(kTestData.length(), sz);
+    uint8_t scratch[kTestData.length()];
+    Slice data(scratch, kTestData.length());
+    ASSERT_OK(reader->Read(0, &data));
+    ASSERT_EQ(kTestData, data);
+  }
+}
+
 } // namespace fs
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager.cc b/src/kudu/fs/block_manager.cc
index 48aea93..d2949d4 100644
--- a/src/kudu/fs/block_manager.cc
+++ b/src/kudu/fs/block_manager.cc
@@ -29,9 +29,26 @@
 #include "kudu/util/flag_tags.h"
 #include "kudu/util/metrics.h"
 
-DEFINE_bool(block_coalesce_close, false,
-            "Coalesce synchronization of data during CloseBlocks()");
-TAG_FLAG(block_coalesce_close, experimental);
+// The default value is optimized for throughput in the case that
+// there are multiple drives backing the tablet. By asynchronously
+// flushing each block before issuing any fsyncs, the IO across
+// disks is done in parallel.
+//
+// This increases throughput but can harm latency in the case that
+// there are few disks and the WAL is on the same disk as the
+// data blocks. The default is chosen based on the assumptions that:
+// - latency is leveled across machines by Raft
+// - latency-sensitive applications can devote a disk to the WAL
+// - super-sensitive applications can devote an SSD to the WAL.
+// - users could always change this to "never", which slows down
+//   throughput but may improve write latency.
+DEFINE_string(block_manager_preflush_control, "finalize",
+              "Controls when to pre-flush a block. Valid values are 'finalize', "
+              "'close', or 'never'. If 'finalize', blocks will be pre-flushed "
+              "when writing is finished. If 'close', blocks will be pre-flushed "
+              "when their transaction is committed. If 'never', blocks will "
+              "never be pre-flushed but still be flushed when closed.");
+TAG_FLAG(block_manager_preflush_control, experimental);
 
 DEFINE_bool(block_manager_lock_dirs, true,
             "Lock the data block directories to prevent concurrent usage. "

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/block_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager.h b/src/kudu/fs/block_manager.h
index 28d4a12..b3f50e9 100644
--- a/src/kudu/fs/block_manager.h
+++ b/src/kudu/fs/block_manager.h
@@ -17,22 +17,19 @@
 
 #pragma once
 
+#include <algorithm>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
-#include <glog/logging.h>
-
-#include "kudu/fs/block_id.h"
 #include "kudu/gutil/ref_counted.h"
-#include "kudu/gutil/stl_util.h"
-#include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/status.h"
 
 namespace kudu {
 
+class BlockId;
 class Env;
 class MemTracker;
 class MetricEntity;
@@ -66,11 +63,12 @@ class Block {
 // Close() is an expensive operation, as it must flush both dirty block data
 // and metadata to disk. The block manager API provides two ways to improve
 // Close() performance:
-// 1. FlushDataAsync() before Close(). If there's enough work to be done
-//    between the two calls, there will be less outstanding I/O to wait for
-//    during Close().
-// 2. CloseBlocks() on a group of blocks. This at least ensures that, when
-//    waiting on outstanding I/O, the waiting is done in parallel.
+// 1. Finalize() before Close(). When 'block_manager_preflush_control' is set
+//    to 'finalize', if there's enough work to be done between the two calls,
+//    there will be less outstanding I/O to wait for during Close().
+// 2. CloseBlocks() on a group of blocks. This ensures: 1) flushing of dirty
+//    blocks are grouped together if possible, resulting in less I/O.
+//    2) when waiting on outstanding I/O, the waiting is done in parallel.
 //
 // NOTE: if a WritableBlock is not explicitly Close()ed, it will be aborted
 // (i.e. deleted).
@@ -83,9 +81,9 @@ class WritableBlock : public Block {
     // There is some dirty data in the block.
     DIRTY,
 
-    // There is an outstanding flush operation asynchronously flushing
-    // dirty block data to disk.
-    FLUSHING,
+    // No more data may be written to the block, but it is not yet guaranteed
+    // to be durably stored on disk.
+    FINALIZED,
 
     // The block is closed. No more operations can be performed on it.
     CLOSED
@@ -119,16 +117,15 @@ class WritableBlock : public Block {
   // outstanding data to reach the disk.
   virtual Status AppendV(const std::vector<Slice>& data) = 0;
 
-  // Begins an asynchronous flush of dirty block data to disk.
-  //
-  // This is purely a performance optimization for Close(); if there is
-  // other work to be done between the final Append() and the future
-  // Close(), FlushDataAsync() will reduce the amount of time spent waiting
-  // for outstanding I/O to complete in Close(). This is analogous to
-  // readahead or prefetching.
+  // Signals that the block will no longer receive writes. Does not guarantee
+  // durability; Close() must still be called for that.
   //
-  // Data may not be written to the block after FlushDataAsync() is called.
-  virtual Status FlushDataAsync() = 0;
+  // When 'block_manager_preflush_control' is set to 'finalize', it also begins an
+  // asynchronous flush of dirty block data to disk. If there is other work
+  // to be done between the final Append() and the future Close(),
+  // Finalize() will reduce the amount of time spent waiting for outstanding
+  // I/O to complete in Close(). This is analogous to readahead or prefetching.
+  virtual Status Finalize() = 0;
 
   // Returns the number of bytes successfully appended via Append().
   virtual size_t BytesAppended() const = 0;
@@ -245,7 +242,8 @@ class BlockManager {
   // Close() for each block but may be optimized for groups of blocks.
   //
   // On success, guarantees that outstanding data is durable.
-  virtual Status CloseBlocks(const std::vector<WritableBlock*>& blocks) = 0;
+  virtual Status CloseBlocks(
+      const std::vector<std::unique_ptr<WritableBlock>>& blocks) = 0;
 
   // Retrieves the IDs of all blocks under management by this block manager.
   // These include ReadableBlocks as well as WritableBlocks.
@@ -260,42 +258,37 @@ class BlockManager {
   virtual FsErrorManager* error_manager() = 0;
 };
 
-// Closes a group of blocks.
-//
-// Blocks must be closed explicitly via CloseBlocks(), otherwise they will
-// be deleted in the in the destructor.
-class ScopedWritableBlockCloser {
+// Group a set of block creations together in a transaction. This has two
+// major motivations: 1) the underlying block manager can optimize
+// synchronization for a batch of blocks if possible to achieve better
+// performance. 2) to be able to track all blocks in one logical operation.
+class BlockTransaction {
  public:
-  ScopedWritableBlockCloser() {}
-
-  ~ScopedWritableBlockCloser() {
-    for (WritableBlock* block : blocks_) {
-      WARN_NOT_OK(block->Abort(), strings::Substitute(
-          "Failed to abort block with id $0", block->id().ToString()));
-    }
-    STLDeleteElements(&blocks_);
+  void AddCreatedBlock(std::unique_ptr<WritableBlock> block) {
+    created_blocks_.emplace_back(std::move(block));
   }
 
-  void AddBlock(std::unique_ptr<WritableBlock> block) {
-    blocks_.push_back(block.release());
-  }
-
-  Status CloseBlocks() {
-    if (blocks_.empty()) {
+  // Commit all the created blocks and close them together.
+  // On success, guarantees that outstanding data is durable.
+  Status CommitCreatedBlocks() {
+    if (created_blocks_.empty()) {
       return Status::OK();
     }
-    ElementDeleter deleter(&blocks_);
 
     // We assume every block is using the same block manager, so any
     // block's manager will do.
-    BlockManager* bm = blocks_[0]->block_manager();
-    return bm->CloseBlocks(blocks_);
+    BlockManager* bm = created_blocks_[0]->block_manager();
+    Status s = bm->CloseBlocks(created_blocks_);
+    if (s.ok()) created_blocks_.clear();
+    return s;
   }
 
-  const std::vector<WritableBlock*>& blocks() const { return blocks_; }
+  const std::vector<std::unique_ptr<WritableBlock>>& created_blocks() const {
+    return created_blocks_;
+  }
 
  private:
-  std::vector<WritableBlock*> blocks_;
+  std::vector<std::unique_ptr<WritableBlock>> created_blocks_;
 };
 
 // Compute an upper bound for a file cache embedded within a block manager

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/file_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/file_block_manager.cc b/src/kudu/fs/file_block_manager.cc
index dfb16e9..bd5d35d 100644
--- a/src/kudu/fs/file_block_manager.cc
+++ b/src/kudu/fs/file_block_manager.cc
@@ -17,6 +17,7 @@
 
 #include "kudu/fs/file_block_manager.h"
 
+#include <algorithm>
 #include <cstddef>
 #include <memory>
 #include <mutex>
@@ -24,7 +25,6 @@
 #include <ostream>
 #include <set>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include <gflags/gflags_declare.h>
@@ -36,6 +36,7 @@
 #include "kudu/fs/error_manager.h"
 #include "kudu/fs/fs_report.h"
 #include "kudu/gutil/bind.h"
+#include "kudu/gutil/casts.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/ref_counted.h"
@@ -47,7 +48,6 @@
 #include "kudu/util/env_util.h"
 #include "kudu/util/file_cache.h"
 #include "kudu/util/malloc.h"
-#include "kudu/util/make_shared.h"
 #include "kudu/util/mem_tracker.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/path_util.h"
@@ -63,9 +63,9 @@ using std::unique_ptr;
 using std::vector;
 using strings::Substitute;
 
-DECLARE_bool(block_coalesce_close);
 DECLARE_bool(block_manager_lock_dirs);
 DECLARE_bool(enable_data_block_fsync);
+DECLARE_string(block_manager_preflush_control);
 
 namespace kudu {
 namespace fs {
@@ -247,7 +247,7 @@ class FileWritableBlock : public WritableBlock {
 
   virtual Status AppendV(const vector<Slice>& data) OVERRIDE;
 
-  virtual Status FlushDataAsync() OVERRIDE;
+  virtual Status Finalize() OVERRIDE;
 
   virtual size_t BytesAppended() const OVERRIDE;
 
@@ -255,6 +255,9 @@ class FileWritableBlock : public WritableBlock {
 
   void HandleError(const Status& s) const;
 
+  // Starts an asynchronous flush of dirty block data to disk.
+  Status FlushDataAsync();
+
  private:
   enum SyncMode {
     SYNC,
@@ -347,14 +350,24 @@ Status FileWritableBlock::AppendV(const vector<Slice>& data) {
 }
 
 Status FileWritableBlock::FlushDataAsync() {
-  DCHECK(state_ == CLEAN || state_ == DIRTY || state_ == FLUSHING)
+  VLOG(3) << "Flushing block " << id();
+  RETURN_NOT_OK_HANDLE_ERROR(writer_->Flush(WritableFile::FLUSH_ASYNC));
+  return Status::OK();
+}
+
+Status FileWritableBlock::Finalize() {
+  DCHECK(state_ == CLEAN || state_ == DIRTY || state_ == FINALIZED)
       << "Invalid state: " << state_;
-  if (state_ == DIRTY) {
-    VLOG(3) << "Flushing block " << id();
-    RETURN_NOT_OK_HANDLE_ERROR(writer_->Flush(WritableFile::FLUSH_ASYNC));
-  }
 
-  state_ = FLUSHING;
+  if (state_ == FINALIZED) {
+    return Status::OK();
+  }
+  VLOG(3) << "Finalizing block " << id();
+  if (state_ == DIRTY &&
+      FLAGS_block_manager_preflush_control == "finalize") {
+    FlushDataAsync();
+  }
+  state_ = FINALIZED;
   return Status::OK();
 }
 
@@ -373,7 +386,7 @@ Status FileWritableBlock::Close(SyncMode mode) {
 
   Status sync;
   if (mode == SYNC &&
-      (state_ == CLEAN || state_ == DIRTY || state_ == FLUSHING)) {
+      (state_ == CLEAN || state_ == DIRTY || state_ == FINALIZED)) {
     // Safer to synchronize data first, then metadata.
     VLOG(3) << "Syncing block " << id();
     if (FLAGS_enable_data_block_fsync) {
@@ -736,19 +749,21 @@ Status FileBlockManager::DeleteBlock(const BlockId& block_id) {
   return Status::OK();
 }
 
-Status FileBlockManager::CloseBlocks(const vector<WritableBlock*>& blocks) {
+Status FileBlockManager::CloseBlocks(const vector<unique_ptr<WritableBlock>>& blocks) {
   VLOG(3) << "Closing " << blocks.size() << " blocks";
-  if (FLAGS_block_coalesce_close) {
+  if (FLAGS_block_manager_preflush_control == "close") {
     // Ask the kernel to begin writing out each block's dirty data. This is
     // done up-front to give the kernel opportunities to coalesce contiguous
     // dirty pages.
-    for (WritableBlock* block : blocks) {
-      RETURN_NOT_OK(block->FlushDataAsync());
+    for (const auto& block : blocks) {
+      internal::FileWritableBlock* fwb =
+          down_cast<internal::FileWritableBlock*>(block.get());
+      RETURN_NOT_OK(fwb->FlushDataAsync());
     }
   }
 
   // Now close each block, waiting for each to become durable.
-  for (WritableBlock* block : blocks) {
+  for (const auto& block : blocks) {
     RETURN_NOT_OK(block->Close());
   }
   return Status::OK();

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/file_block_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/file_block_manager.h b/src/kudu/fs/file_block_manager.h
index 7e2de97..fb79475 100644
--- a/src/kudu/fs/file_block_manager.h
+++ b/src/kudu/fs/file_block_manager.h
@@ -88,7 +88,7 @@ class FileBlockManager : public BlockManager {
 
   Status DeleteBlock(const BlockId& block_id) override;
 
-  Status CloseBlocks(const std::vector<WritableBlock*>& blocks) override;
+  Status CloseBlocks(const std::vector<std::unique_ptr<WritableBlock>>& blocks) override;
 
   Status GetAllBlockIds(std::vector<BlockId>* block_ids) override;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/log_block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager-test.cc b/src/kudu/fs/log_block_manager-test.cc
index e33e773..d7844ed 100644
--- a/src/kudu/fs/log_block_manager-test.cc
+++ b/src/kudu/fs/log_block_manager-test.cc
@@ -284,7 +284,7 @@ TEST_F(LogBlockManagerTest, MetricsTest) {
   BlockId saved_id;
   {
     Random rand(SeedRandom());
-    ScopedWritableBlockCloser closer;
+    BlockTransaction transaction;
     for (int i = 0; i < 10; i++) {
       unique_ptr<WritableBlock> b;
       ASSERT_OK(bm_->CreateBlock(test_block_opts_, &b));
@@ -296,12 +296,13 @@ TEST_F(LogBlockManagerTest, MetricsTest) {
         data[i] = rand.Next();
       }
       b->Append(Slice(data, sizeof(data)));
-      closer.AddBlock(std::move(b));
+      ASSERT_OK(b->Finalize());
+      transaction.AddCreatedBlock(std::move(b));
     }
-    ASSERT_NO_FATAL_FAILURE(CheckLogMetrics(entity, 0, 1, 10, 0));
+    // Metrics for full containers are updated after Finalize().
+    ASSERT_NO_FATAL_FAILURE(CheckLogMetrics(entity, 0, 1, 10, 10));
 
-    // Only when the blocks are closed are the containers considered full.
-    ASSERT_OK(closer.CloseBlocks());
+    ASSERT_OK(transaction.CommitCreatedBlocks());
     ASSERT_NO_FATAL_FAILURE(CheckLogMetrics(entity, 10 * 1024, 11, 10, 10));
   }
 
@@ -370,14 +371,14 @@ TEST_F(LogBlockManagerTest, TestReuseBlockIds) {
 
   // Create 4 containers, with the first four block IDs in the sequence.
   {
-    ScopedWritableBlockCloser closer;
+    BlockTransaction transaction;
     for (int i = 0; i < 4; i++) {
       unique_ptr<WritableBlock> writer;
       ASSERT_OK(bm_->CreateBlock(test_block_opts_, &writer));
       block_ids.push_back(writer->id());
-      closer.AddBlock(std::move(writer));
+      transaction.AddCreatedBlock(std::move(writer));
     }
-    ASSERT_OK(closer.CloseBlocks());
+    ASSERT_OK(transaction.CommitCreatedBlocks());
   }
 
   // Create one more block, which should reuse the first container.
@@ -935,13 +936,14 @@ TEST_F(LogBlockManagerTest, TestRepairPreallocateExcessSpace) {
 
   // Create several full containers.
   {
-    ScopedWritableBlockCloser closer;
+    BlockTransaction transaction;
     for (int i = 0; i < kNumContainers; i++) {
       unique_ptr<WritableBlock> block;
       ASSERT_OK(bm_->CreateBlock(test_block_opts_, &block));
       ASSERT_OK(block->Append("a"));
-      closer.AddBlock(std::move(block));
+      transaction.AddCreatedBlock(std::move(block));
     }
+    ASSERT_OK(transaction.CommitCreatedBlocks());
   }
   vector<string> container_names;
   NO_FATALS(GetContainerNames(&container_names));
@@ -1127,12 +1129,12 @@ TEST_F(LogBlockManagerTest, TestRepairPartialRecords) {
 
   // Create some containers.
   {
-    ScopedWritableBlockCloser closer;
+    BlockTransaction transaction;
     for (int i = 0; i < kNumContainers; i++) {
       unique_ptr<WritableBlock> block;
       ASSERT_OK(bm_->CreateBlock(test_block_opts_, &block));
       ASSERT_OK(block->Append("a"));
-      closer.AddBlock(std::move(block));
+      transaction.AddCreatedBlock(std::move(block));
     }
   }
   vector<string> container_names;
@@ -1344,5 +1346,48 @@ TEST_F(LogBlockManagerTest, TestOpenWithFailedDirectories) {
   FLAGS_env_inject_eio = 0;
 }
 
+// Test Close() a FINALIZED block. Including,
+// 1) a container can be reused when the block is finalized.
+// 2) the block cannot be opened/found until close it.
+// 3) the same container is not marked as available twice.
+TEST_F(LogBlockManagerTest, TestFinalizeBlock) {
+  // Create 4 blocks.
+  vector<unique_ptr<WritableBlock>> blocks;
+  for (int i = 0; i < 4; i++) {
+    unique_ptr<WritableBlock> writer;
+    ASSERT_OK(bm_->CreateBlock(test_block_opts_, &writer));
+    ASSERT_OK(writer->Append("test data"));
+    ASSERT_OK(writer->Finalize());
+    blocks.emplace_back(std::move(writer));
+  }
+  ASSERT_EQ(1, bm_->all_containers_by_name_.size());
+
+  for (const auto& block : blocks) {
+    // Open the block and verify they cannot be found.
+    ASSERT_TRUE(bm_->OpenBlock(block->id(), nullptr).IsNotFound());
+    ASSERT_OK(block->Close());
+  }
+
+  ASSERT_EQ(1, bm_->all_containers_by_name_.size());
+  // Ensure the same container has not been marked as available twice.
+  ASSERT_EQ(1, bm_->available_containers_by_data_dir_.begin()->second.size());
+}
+
+TEST_F(LogBlockManagerTest, TestAbortBlock) {
+  unique_ptr<WritableBlock> writer;
+  ASSERT_OK(bm_->CreateBlock(test_block_opts_, &writer));
+  ASSERT_OK(writer->Append("test data"));
+  ASSERT_OK(writer->Abort());
+  // Ensures the container is available after block's Abort().
+  ASSERT_EQ(1, bm_->available_containers_by_data_dir_.begin()->second.size());
+
+  ASSERT_OK(bm_->CreateBlock(test_block_opts_, &writer));
+  ASSERT_OK(writer->Append("test data"));
+  ASSERT_OK(writer->Finalize());
+  ASSERT_OK(writer->Abort());
+  // Ensures the container is available after block's Abort().
+  ASSERT_EQ(1, bm_->available_containers_by_data_dir_.begin()->second.size());
+}
+
 } // namespace fs
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index b733d23..cf80d4e 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -18,15 +18,16 @@
 #include "kudu/fs/log_block_manager.h"
 
 #include <algorithm>
-#include <cerrno>
 #include <cstddef>
 #include <cstdint>
+#include <errno.h>
 #include <map>
 #include <memory>
 #include <mutex>
 #include <numeric>
 #include <ostream>
 #include <set>
+#include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -45,6 +46,7 @@
 #include "kudu/gutil/bind.h"
 #include "kudu/gutil/bind_helpers.h"
 #include "kudu/gutil/callback.h"
+#include "kudu/gutil/casts.h"
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/stl_util.h"
@@ -61,7 +63,6 @@
 #include "kudu/util/file_cache.h"
 #include "kudu/util/flag_tags.h"
 #include "kudu/util/locks.h"
-#include "kudu/util/make_shared.h"
 #include "kudu/util/malloc.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
@@ -75,8 +76,8 @@
 #include "kudu/util/trace.h"
 
 DECLARE_bool(block_manager_lock_dirs);
-DECLARE_bool(block_coalesce_close);
 DECLARE_bool(enable_data_block_fsync);
+DECLARE_string(block_manager_preflush_control);
 
 // TODO(unknown): How should this be configured? Should provide some guidance.
 DEFINE_uint64(log_container_max_size, 10LU * 1024 * 1024 * 1024,
@@ -138,6 +139,7 @@ namespace fs {
 
 using internal::LogBlock;
 using internal::LogBlockContainer;
+using internal::LogWritableBlock;
 using pb_util::ReadablePBContainerFile;
 using pb_util::WritablePBContainerFile;
 using std::map;
@@ -246,11 +248,6 @@ class LogBlock : public RefCountedThreadSafe<LogBlock> {
 // persisted.
 class LogWritableBlock : public WritableBlock {
  public:
-  enum SyncMode {
-    SYNC,
-    NO_SYNC
-  };
-
   LogWritableBlock(LogBlockContainer* container, BlockId block_id,
                    int64_t block_offset);
 
@@ -268,21 +265,29 @@ class LogWritableBlock : public WritableBlock {
 
   virtual Status AppendV(const vector<Slice>& data) OVERRIDE;
 
-  virtual Status FlushDataAsync() OVERRIDE;
+  virtual Status Finalize() OVERRIDE;
 
   virtual size_t BytesAppended() const OVERRIDE;
 
   virtual State state() const OVERRIDE;
 
-  // Actually close the block, possibly synchronizing its dirty data and
-  // metadata to disk.
-  Status DoClose(SyncMode mode);
+  // Actually close the block, finalizing it if it has not yet been
+  // finalized. Also updates various metrics.
+  //
+  // This is called after synchronization of dirty data and metadata
+  // to disk.
+  void DoClose();
+
+  // Starts an asynchronous flush of dirty block data to disk.
+  Status FlushDataAsync();
 
   // Write this block's metadata to disk.
   //
   // Does not synchronize the written data; that takes place in Close().
   Status AppendMetadata();
 
+  LogBlockContainer* container() const { return container_; }
+
  private:
   // The owning container. Must outlive the block.
   LogBlockContainer* container_;
@@ -318,6 +323,11 @@ class LogWritableBlock : public WritableBlock {
 // functions are marked explicitly.
 class LogBlockContainer {
  public:
+  enum SyncMode {
+    SYNC,
+    NO_SYNC
+  };
+
   static const char* kMagic;
 
   // Creates a new block container in 'dir'.
@@ -340,15 +350,11 @@ class LogBlockContainer {
                      const std::string& id,
                      unique_ptr<LogBlockContainer>* container);
 
-  // Indicates that the writing of 'block' is finished. If successful,
-  // adds the block to the block manager's in-memory maps.
+  // Closes a set of blocks belonging to this container, possibly synchronizing
+  // the dirty data and metadata to disk.
   //
-  // Returns a status that is either the same as 's' (if !s.ok()) or
-  // potentially different (if s.ok() and FinishBlock() failed).
-  //
-  // After returning, this container has been released to the block manager
-  // and may no longer be used in the context of writing 'block'.
-  Status FinishBlock(const Status& s, WritableBlock* block);
+  // If successful, adds all blocks to the block manager's in-memory maps.
+  Status DoCloseBlocks(const vector<LogWritableBlock*>& blocks, SyncMode mode);
 
   // Frees the space associated with a block at 'offset' and 'length'. This
   // is a physical operation, not a logical one; a separate AppendMetadata()
@@ -439,21 +445,21 @@ class LogBlockContainer {
       std::vector<scoped_refptr<internal::LogBlock>>* dead_blocks,
       uint64_t* max_block_id);
 
-  // Updates internal bookkeeping state to reflect the creation of a block,
-  // marking this container as full if needed. Should only be called when a
-  // block is fully written, as it will round up the container data file's position.
-  //
-  // This function is thread unsafe.
+  // Updates internal bookkeeping state to reflect the creation of a block.
   void BlockCreated(const scoped_refptr<LogBlock>& block);
 
   // Updates internal bookkeeping state to reflect the deletion of a block.
   //
-  // Unlike BlockCreated(), this function is thread safe because block
-  // deletions can happen concurrently with creations.
+  // This function is thread safe because block deletions can happen concurrently
+  // with creations.
   //
   // Note: the container is not made "unfull"; containers remain sparse until deleted.
   void BlockDeleted(const scoped_refptr<LogBlock>& block);
 
+  // Finalizes a fully written block. It updates the container data file's position,
+  // truncates the container if full and marks the container as available.
+  void FinalizeBlock(int64_t block_offset, int64_t block_length);
+
   // Run a task on this container's data directory thread pool.
   //
   // Normally the task is performed asynchronously. However, if submission to
@@ -471,15 +477,15 @@ class LogBlockContainer {
 
   // Simple accessors.
   LogBlockManager* block_manager() const { return block_manager_; }
-  int64_t next_block_offset() const { return next_block_offset_; }
-  int64_t total_bytes() const { return total_bytes_; }
-  int64_t total_blocks() const { return total_blocks_; }
+  int64_t next_block_offset() const { return next_block_offset_.Load(); }
+  int64_t total_bytes() const { return total_bytes_.Load(); }
+  int64_t total_blocks() const { return total_blocks_.Load(); }
   int64_t live_bytes() const { return live_bytes_.Load(); }
   int64_t live_bytes_aligned() const { return live_bytes_aligned_.Load(); }
   int64_t live_blocks() const { return live_blocks_.Load(); }
   bool full() const {
-    return next_block_offset_ >= FLAGS_log_container_max_size ||
-        (max_num_blocks_ && (total_blocks_ >= max_num_blocks_));
+    return next_block_offset() >= FLAGS_log_container_max_size ||
+        (max_num_blocks_ && (total_blocks() >= max_num_blocks_));
   }
   const LogBlockManagerMetrics* metrics() const { return metrics_; }
   DataDir* data_dir() const { return data_dir_; }
@@ -514,6 +520,14 @@ class LogBlockContainer {
       uint64_t* data_file_size,
       uint64_t* max_block_id);
 
+  // Updates this container data file's position based on the offset and length
+  // of a block, marking this container as full if needed. Should only be called
+  // when a block is fully written, as it will round up the container data file's
+  // position.
+  //
+  // This function is thread unsafe.
+  void UpdateNextBlockOffset(int64_t block_offset, int64_t block_length);
+
   // The owning block manager. Must outlive the container itself.
   LogBlockManager* const block_manager_;
 
@@ -530,28 +544,22 @@ class LogBlockContainer {
   shared_ptr<RWFile> data_file_;
 
   // The offset of the next block to be written to the container.
-  int64_t next_block_offset_ = 0;
+  AtomicInt<int64_t> next_block_offset_;
 
   // The amount of data (post block alignment) written thus far to the container.
-  int64_t total_bytes_ = 0;
+  AtomicInt<int64_t> total_bytes_;
 
   // The number of blocks written thus far in the container.
-  int64_t total_blocks_ = 0;
+  AtomicInt<int64_t> total_blocks_;
 
   // The amount of data present in not-yet-deleted blocks of the container.
-  //
-  // Declared atomic because it is mutated from BlockDeleted().
   AtomicInt<int64_t> live_bytes_;
 
   // The amount of data (post block alignment) present in not-yet-deleted
   // blocks of the container.
-  //
-  // Declared atomic because it is mutated from BlockDeleted().
   AtomicInt<int64_t> live_bytes_aligned_;
 
   // The number of not-yet-deleted blocks in the container.
-  //
-  // Declared atomic because it is mutated from BlockDeleted().
   AtomicInt<int64_t> live_blocks_;
 
   // The metrics. Not owned by the log container; it has the same lifespan
@@ -577,6 +585,9 @@ LogBlockContainer::LogBlockContainer(
                                 data_dir)),
       metadata_file_(std::move(metadata_file)),
       data_file_(std::move(data_file)),
+      next_block_offset_(0),
+      total_bytes_(0),
+      total_blocks_(0),
       live_bytes_(0),
       live_bytes_aligned_(0),
       live_blocks_(0),
@@ -725,8 +736,8 @@ Status LogBlockContainer::Open(LogBlockManager* block_manager,
 Status LogBlockContainer::TruncateDataToNextBlockOffset() {
   if (full()) {
     VLOG(2) << Substitute("Truncating container $0 to offset $1",
-                          ToString(), next_block_offset_);
-    RETURN_NOT_OK_HANDLE_ERROR(data_file_->Truncate(next_block_offset_));
+                          ToString(), next_block_offset());
+    RETURN_NOT_OK_HANDLE_ERROR(data_file_->Truncate(next_block_offset()));
   }
   return Status::OK();
 }
@@ -836,6 +847,7 @@ Status LogBlockContainer::ProcessRecord(
       //
       // If we ignored deleted blocks, we would end up reusing the space
       // belonging to the last deleted block in the container.
+      UpdateNextBlockOffset(lb->offset(), lb->length());
       BlockCreated(lb);
 
       (*live_block_records)[block_id].Swap(record);
@@ -866,55 +878,45 @@ Status LogBlockContainer::ProcessRecord(
   return Status::OK();
 }
 
-Status LogBlockContainer::FinishBlock(const Status& s, WritableBlock* block) {
-  Status result_status = s;
-  auto cleanup = MakeScopedCleanup([&]() {
-    // Handle errors that have not been handled by marking the
-    // container as read-only to forbid future writes. Because
-    // the on-disk state may contain partial/complete data/metadata
-    // at this point, it is not safe to either overwrite it or
-    // append to it.
-    if (!result_status.ok()) {
-      SetReadOnly();
+Status LogBlockContainer::DoCloseBlocks(const vector<LogWritableBlock*>& blocks,
+                                        SyncMode mode) {
+  DCHECK(!read_only());
+  auto sync_blocks = [&]() -> Status {
+    if (mode == SYNC) {
+      VLOG(3) << "Syncing data file " << data_file_->filename();
+      RETURN_NOT_OK(SyncData());
     }
 
-    block_manager_->MakeContainerAvailable(this);
-  });
-
-  // Early return; 'cleanup' marks the container as read-only.
-  RETURN_NOT_OK(s);
+    // Append metadata only after data is synced so that there's
+    // no chance of metadata landing on the disk before the data.
+    for (auto* block : blocks) {
+      RETURN_NOT_OK_PREPEND(block->AppendMetadata(),
+                            "unable to append block's metadata during close");
+    }
 
-  // A failure when syncing the container means the container (and its new
-  // blocks) may be missing the next time the on-disk state is reloaded. As
-  // such, it's not correct to add the block to in-memory state unless
-  // synchronization succeeds.
-  //
-  // On the other hand, the new blocks' metadata may be already on disk
-  // despite syncing failure.
-  //
-  // Thus, we consider the failure as fatal and mark the container
-  // 'read only'. See 'cleanup' above.
-  result_status = block_manager()->SyncContainer(*this);
-  RETURN_NOT_OK(result_status);
+    if (mode == SYNC) {
+      VLOG(3) << "Syncing metadata file " << metadata_file_->filename();
+      RETURN_NOT_OK(SyncMetadata());
+    }
 
-  scoped_refptr<LogBlock> lb = block_manager()->AddLogBlock(
-      this, block->id(), next_block_offset_, block->BytesAppended());
-  CHECK(lb);
-  BlockCreated(lb);
+    RETURN_NOT_OK(block_manager()->SyncContainer(*this));
 
-  // Truncate the container if it's now full; any left over preallocated space
-  // is no longer needed.
-  //
-  // Note that this take places _after_ the container has been synced to disk.
-  // That's OK; truncation isn't needed for correctness, and in the event of a
-  // crash or error, it will be retried at startup.
-  WARN_NOT_OK(TruncateDataToNextBlockOffset(),
-              "could not truncate excess preallocated space");
+    for (LogWritableBlock* block : blocks) {
+      if (blocks.size() > 1) DCHECK_EQ(block->state(), WritableBlock::State::FINALIZED);
+      block->DoClose();
+    }
+    return Status::OK();
+  };
 
-  if (full() && block_manager()->metrics()) {
-    block_manager()->metrics()->full_containers->Increment();
+  Status s = sync_blocks();
+  if (!s.ok()) {
+    // Mark container as read-only to forbid further writes in case of
+    // failure. Because the on-disk state may contain partial/complete
+    // data/metadata at this point, it is not safe to either overwrite
+    // it or append to it.
+    SetReadOnly();
   }
-  return Status::OK();
+  return s;
 }
 
 Status LogBlockContainer::PunchHole(int64_t offset, int64_t length) {
@@ -936,7 +938,7 @@ Status LogBlockContainer::WriteData(int64_t offset, const Slice& data) {
 
 Status LogBlockContainer::WriteVData(int64_t offset, const vector<Slice>& data) {
   DCHECK(!read_only());
-  DCHECK_GE(offset, next_block_offset_);
+  DCHECK_GE(offset, next_block_offset());
 
   RETURN_NOT_OK_HANDLE_ERROR(data_file_->WriteV(offset, data));
 
@@ -1041,9 +1043,29 @@ Status LogBlockContainer::EnsurePreallocated(int64_t block_start_offset,
   return Status::OK();
 }
 
-void LogBlockContainer::BlockCreated(const scoped_refptr<LogBlock>& block) {
+void LogBlockContainer::FinalizeBlock(int64_t block_offset, int64_t block_length) {
+  // Updates this container's next block offset before marking it as available
+  // to ensure thread safety while updating internal bookkeeping state.
+  UpdateNextBlockOffset(block_offset, block_length);
+
+  // Truncate the container if it's now full; any left over preallocated space
+  // is no longer needed.
+  //
+  // Note that depending on when FinalizeBlock() is called, this can take place
+  // _after_ the container has been synced to disk. That's OK; truncation isn't
+  // needed for correctness, and in the event of a crash or error, it will be
+  // retried at startup.
+  WARN_NOT_OK(TruncateDataToNextBlockOffset(),
+              "could not truncate excess preallocated space");
+  if (full() && block_manager_->metrics()) {
+    block_manager_->metrics()->full_containers->Increment();
+  }
+  block_manager_->MakeContainerAvailable(this);
+}
+
+void LogBlockContainer::UpdateNextBlockOffset(int64_t block_offset, int64_t block_length) {
   DCHECK(!read_only());
-  DCHECK_GE(block->offset(), 0);
+  DCHECK_GE(block_offset, 0);
 
   // The log block manager maintains block contiguity as an invariant, which
   // means accounting for the new block should be as simple as adding its
@@ -1056,29 +1078,28 @@ void LogBlockContainer::BlockCreated(const scoped_refptr<LogBlock>& block) {
   // boundary. This guarantees that the disk space can be reclaimed when
   // the block is deleted.
   int64_t new_next_block_offset = KUDU_ALIGN_UP(
-      block->offset() + block->length(),
+      block_offset + block_length,
       instance()->filesystem_block_size_bytes());
-  if (PREDICT_FALSE(new_next_block_offset < next_block_offset_)) {
-    LOG(WARNING) << Substitute(
-        "Container $0 unexpectedly tried to lower its next block offset "
-        "(from $1 to $2), ignoring",
-        ToString(), next_block_offset_, new_next_block_offset);
-  } else {
-    next_block_offset_ = new_next_block_offset;
-  }
-  total_bytes_ += block->fs_aligned_length();
-  total_blocks_++;
-  live_bytes_.IncrementBy(block->length());
-  live_bytes_aligned_.IncrementBy(block->fs_aligned_length());
-  live_blocks_.Increment();
+  next_block_offset_.StoreMax(new_next_block_offset);
 
   if (full()) {
     VLOG(1) << Substitute(
         "Container $0 with size $1 is now full, max size is $2",
-        ToString(), next_block_offset_, FLAGS_log_container_max_size);
+        ToString(), next_block_offset(), FLAGS_log_container_max_size);
   }
 }
 
+void LogBlockContainer::BlockCreated(const scoped_refptr<LogBlock>& block) {
+  DCHECK(!read_only());
+  DCHECK_GE(block->offset(), 0);
+
+  total_bytes_.IncrementBy(block->fs_aligned_length());
+  total_blocks_.Increment();
+  live_bytes_.IncrementBy(block->length());
+  live_bytes_aligned_.IncrementBy(block->fs_aligned_length());
+  live_blocks_.Increment();
+}
+
 void LogBlockContainer::BlockDeleted(const scoped_refptr<LogBlock>& block) {
   DCHECK(!read_only());
   DCHECK_GE(block->offset(), 0);
@@ -1189,18 +1210,52 @@ LogWritableBlock::~LogWritableBlock() {
 }
 
 Status LogWritableBlock::Close() {
-  return DoClose(SYNC);
+  return container_->DoCloseBlocks({ this }, LogBlockContainer::SyncMode::SYNC);
 }
 
 Status LogWritableBlock::Abort() {
-  // Abort the operation for read-only container.
+  // Only updates metrics and block state for read-only container.
   if (container_->read_only()) {
+    if (state_ != CLOSED) {
+      state_ = CLOSED;
+      if (container_->metrics()) {
+        container_->metrics()->generic_metrics.blocks_open_writing->Decrement();
+        container_->metrics()->generic_metrics.total_bytes_written->IncrementBy(
+            block_length_);
+      }
+    }
     return Status::Aborted("container $0 is read-only", container_->ToString());
   }
 
-  RETURN_NOT_OK(DoClose(NO_SYNC));
-
-  // DoClose() has unlocked the container; it may be locked by someone else.
+  // Close the block and then delete it. Theoretically, we could do nothing
+  // for Abort() other than updating metrics and block state. Here is the
+  // reasoning why it would be safe to do so. Currently, failures in block
+  // creation can happen at various stages:
+  // 1) before the block is finalized (either in CLEAN or DIRTY state). It is
+  //    safe to do nothing, as the block is not yet finalized and next block
+  //    will overwrite the dirty data.
+  // 2) after the block is finalized but before the metadata is successfully
+  //    appended. That means the container's internal bookkeeping has been
+  //    updated and the data could be durable. Doing nothing can result in
+  //    data-consuming gaps in containers, but these gaps can be cleaned up
+  //    by hole repunching at start up.
+  // 3) after metadata is appended. If we do nothing, this can result in
+  //    orphaned blocks if the metadata is durable. But orphaned blocks can be
+  //    garbage collected later.
+  //
+  // TODO(Hao): implement a fine-grained abort handling.
+  // However it is better to provide a fine-grained abort handling to
+  // avoid large chunks of data-consuming gaps and orphaned blocks. A
+  // possible way to do so is,
+  // 1) for per block abort, if the block state is FINALIZED, do hole
+  //    punching.
+  // 2) for BlockTransaction abort, DoCloseBlock() should append metadata
+  //    for deletion and coalescing hole punching for blocks in the same
+  //    container.
+
+  RETURN_NOT_OK(container_->DoCloseBlocks({ this }, LogBlockContainer::SyncMode::NO_SYNC));
+
+  // DoCloseBlocks() has unlocked the container; it may be locked by someone else.
   // But block_manager_ is immutable, so this is safe.
   return container_->block_manager()->DeleteBlock(id());
 }
@@ -1219,7 +1274,15 @@ Status LogWritableBlock::Append(const Slice& data) {
 
 Status LogWritableBlock::AppendV(const vector<Slice>& data) {
   DCHECK(state_ == CLEAN || state_ == DIRTY)
-  << "Invalid state: " << state_;
+      << "Invalid state: " << state_;
+
+  // This could be true if Finalize() is successful but DoCloseBlocks() is not.
+  // That means the container is read-only but marked as available. Another
+  // block can actually try to write to it.
+  if (container_->read_only()) {
+    return Status::IllegalState(Substitute("container $0 is read-only",
+                                           container_->ToString()));
+  }
 
   // Calculate the amount of data to write
   size_t data_size = accumulate(data.begin(), data.end(), static_cast<size_t>(0),
@@ -1227,10 +1290,8 @@ Status LogWritableBlock::AppendV(const vector<Slice>& data) {
                                   return sum + curr.size();
                                 });
 
-  // The metadata change is deferred to Close() or FlushDataAsync(),
-  // whichever comes first. We can't do it now because the block's
-  // length is still in flux.
-
+  // The metadata change is deferred to Close(). We can't do
+  // it now because the block's length is still in flux.
   int64_t cur_block_offset = block_offset_ + block_length_;
   RETURN_NOT_OK(container_->EnsurePreallocated(cur_block_offset, data_size));
 
@@ -1249,34 +1310,32 @@ Status LogWritableBlock::AppendV(const vector<Slice>& data) {
 }
 
 Status LogWritableBlock::FlushDataAsync() {
-  Status s;
-  auto cleanup = MakeScopedCleanup([&]() {
-    // Mark container as read-only to avoid further
-    // writes in case of failure, as it is not safe to
-    // either overwrite the unsuccessful writes or
-    // append to it. Note that we do not care the failure
-    // of FlushData(), since the corresponding metadata
-    // has not been appended yet.
-    if (!s.ok()) {
-      container_->SetReadOnly();
-    }
-  });
+  VLOG(3) << "Flushing block " << id();
+  RETURN_NOT_OK(container_->FlushData(block_offset_, block_length_));
+  return Status::OK();
+}
 
-  DCHECK(state_ == CLEAN || state_ == DIRTY || state_ == FLUSHING)
+Status LogWritableBlock::Finalize() {
+  DCHECK(state_ == CLEAN || state_ == DIRTY || state_ == FINALIZED)
       << "Invalid state: " << state_;
-  if (state_ == DIRTY) {
-    VLOG(3) << "Flushing block " << id();
-    RETURN_NOT_OK(container_->FlushData(block_offset_, block_length_));
 
-    s = AppendMetadata();
-    RETURN_NOT_OK_PREPEND(s, "Unable to append block metadata");
+  if (state_ == FINALIZED) {
+    return Status::OK();
+  }
+
+  auto cleanup = MakeScopedCleanup([&]() {
+    container_->FinalizeBlock(block_offset_, block_length_);
+    state_ = FINALIZED;
+  });
 
-    // TODO(unknown): Flush just the range we care about.
-    s = container_->FlushMetadata();
-    RETURN_NOT_OK(s);
+  VLOG(3) << "Finalizing block " << id();
+  if (state_ == DIRTY &&
+      FLAGS_block_manager_preflush_control == "finalize") {
+    // We do not mark the container as read-only if FlushDataAsync() fails
+    // since the corresponding metadata has not yet been appended.
+    RETURN_NOT_OK(FlushDataAsync());
   }
 
-  state_ = FLUSHING;
   return Status::OK();
 }
 
@@ -1284,56 +1343,30 @@ size_t LogWritableBlock::BytesAppended() const {
   return block_length_;
 }
 
-
 WritableBlock::State LogWritableBlock::state() const {
   return state_;
 }
 
-Status LogWritableBlock::DoClose(SyncMode mode) {
-  if (state_ == CLOSED) {
-    return Status::OK();
-  }
+void LogWritableBlock::DoClose() {
+  if (state_ == CLOSED) return;
 
-  // Tracks the first failure (if any).
-  //
-  // It's important that any subsequent failures mutate 's' before
-  // returning. Otherwise 'cleanup' won't properly provide the first
-  // failure to LogBlockContainer::FinishBlock().
-  //
-  // Note also that when 'cleanup' goes out of scope it may mutate 's'.
-  Status s;
-  {
-    auto cleanup = MakeScopedCleanup([&]() {
-        if (container_->metrics()) {
-          container_->metrics()->generic_metrics.blocks_open_writing->Decrement();
-          container_->metrics()->generic_metrics.total_bytes_written->IncrementBy(
-              BytesAppended());
-        }
-
-        state_ = CLOSED;
-        s = container_->FinishBlock(s, this);
-      });
-    // FlushDataAsync() was not called; append the metadata now.
-    if (state_ == CLEAN || state_ == DIRTY) {
-      s = AppendMetadata();
-      RETURN_NOT_OK_PREPEND(s, "Unable to flush block during close");
-    }
-
-    if (mode == SYNC &&
-        (state_ == CLEAN || state_ == DIRTY || state_ == FLUSHING)) {
-      VLOG(3) << "Syncing block " << id();
-
-      // TODO(unknown): Sync just this block's dirty data.
-      s = container_->SyncData();
-      RETURN_NOT_OK(s);
+  if (container_->metrics()) {
+    container_->metrics()->generic_metrics.blocks_open_writing->Decrement();
+    container_->metrics()->generic_metrics.total_bytes_written->IncrementBy(
+        block_length_);
+  }
 
-      // TODO(unknown): Sync just this block's dirty metadata.
-      s = container_->SyncMetadata();
-      RETURN_NOT_OK(s);
-    }
+  // Finalize() was not called; this indicates we should
+  // finalize the block.
+  if (state_ == CLEAN || state_ == DIRTY) {
+    container_->FinalizeBlock(block_offset_, block_length_);
   }
 
-  return s;
+  scoped_refptr<LogBlock> lb = container_->block_manager()->AddLogBlock(
+      container_, block_id_, block_offset_, block_length_);
+  CHECK(lb);
+  container_->BlockCreated(lb);
+  state_ = CLOSED;
 }
 
 Status LogWritableBlock::AppendMetadata() {
@@ -1659,9 +1692,9 @@ Status LogBlockManager::CreateBlock(const CreateBlockOptions& opts,
     new_block_id.SetId(next_block_id_.Increment());
   } while (!TryUseBlockId(new_block_id));
 
-  block->reset(new internal::LogWritableBlock(container,
-                                              new_block_id,
-                                              container->next_block_offset()));
+  block->reset(new LogWritableBlock(container,
+                                    new_block_id,
+                                    container->next_block_offset()));
   VLOG(3) << "Created block " << (*block)->id() << " in container "
           << container->ToString();
   return Status::OK();
@@ -1699,8 +1732,8 @@ Status LogBlockManager::DeleteBlock(const BlockId& block_id) {
 
     lb = it->second;
     if (lb->container()->read_only()) {
-      return Status::IllegalState("container $0 is read-only",
-                                  lb->container()->ToString());
+      return Status::IllegalState(Substitute("container $0 is read-only",
+                                             lb->container()->ToString()));
     }
 
     // Return early if deleting a block in a failed directory.
@@ -1741,20 +1774,28 @@ Status LogBlockManager::DeleteBlock(const BlockId& block_id) {
   return Status::OK();
 }
 
-Status LogBlockManager::CloseBlocks(const std::vector<WritableBlock*>& blocks) {
+Status LogBlockManager::CloseBlocks(const std::vector<unique_ptr<WritableBlock>>& blocks) {
   VLOG(3) << "Closing " << blocks.size() << " blocks";
-  if (FLAGS_block_coalesce_close) {
-    // Ask the kernel to begin writing out each block's dirty data. This is
-    // done up-front to give the kernel opportunities to coalesce contiguous
-    // dirty pages.
-    for (WritableBlock* block : blocks) {
-      RETURN_NOT_OK(block->FlushDataAsync());
+
+  unordered_map<LogBlockContainer*, vector<LogWritableBlock*>> created_block_map;
+  for (const auto& block : blocks) {
+    LogWritableBlock* lwb = down_cast<LogWritableBlock*>(block.get());
+
+    if (FLAGS_block_manager_preflush_control == "close") {
+      // Ask the kernel to begin writing out each block's dirty data. This is
+      // done up-front to give the kernel opportunities to coalesce contiguous
+      // dirty pages.
+      RETURN_NOT_OK(lwb->FlushDataAsync());
     }
+    created_block_map[lwb->container()].emplace_back(lwb);
   }
 
-  // Now close each block, waiting for each to become durable.
-  for (WritableBlock* block : blocks) {
-    RETURN_NOT_OK(block->Close());
+  // Close all blocks and sync the blocks belonging to the same
+  // container together to reduce fsync() usage, waiting for them
+  // to become durable.
+  for (const auto& entry : created_block_map) {
+    RETURN_NOT_OK(entry.first->DoCloseBlocks(entry.second,
+                                             LogBlockContainer::SyncMode::SYNC));
   }
   return Status::OK();
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/fs/log_block_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.h b/src/kudu/fs/log_block_manager.h
index 4aa61d8..71afa20 100644
--- a/src/kudu/fs/log_block_manager.h
+++ b/src/kudu/fs/log_block_manager.h
@@ -57,6 +57,7 @@ struct FsReport;
 namespace internal {
 class LogBlock;
 class LogBlockContainer;
+class LogWritableBlock;
 
 struct LogBlockManagerMetrics;
 } // namespace internal
@@ -150,8 +151,6 @@ struct LogBlockManagerMetrics;
 // - Implement garbage collection fallback for hole punching.
 // - Implement locality hints so that specific containers can be used for
 //   groups of blocks (i.e. an entire column).
-// - Unlock containers on FlushDataAsync() so that the workflow in
-//   BlockManagerTest::CloseManyBlocksTest can use just one container.
 // - Implement failure recovery (i.e. metadata truncation and other
 //   similarly recoverable errors).
 // - Evaluate and implement a solution for data integrity (e.g. per-block
@@ -181,19 +180,23 @@ class LogBlockManager : public BlockManager {
 
   Status DeleteBlock(const BlockId& block_id) override;
 
-  Status CloseBlocks(const std::vector<WritableBlock*>& blocks) override;
+  Status CloseBlocks(const std::vector<std::unique_ptr<WritableBlock>>& blocks) override;
 
   Status GetAllBlockIds(std::vector<BlockId>* block_ids) override;
 
   FsErrorManager* error_manager() override { return error_manager_; }
 
  private:
+  FRIEND_TEST(LogBlockManagerTest, TestAbortBlock);
+  FRIEND_TEST(LogBlockManagerTest, TestCloseFinalizedBlock);
+  FRIEND_TEST(LogBlockManagerTest, TestFinalizeBlock);
   FRIEND_TEST(LogBlockManagerTest, TestLookupBlockLimit);
   FRIEND_TEST(LogBlockManagerTest, TestMetadataTruncation);
   FRIEND_TEST(LogBlockManagerTest, TestParseKernelRelease);
   FRIEND_TEST(LogBlockManagerTest, TestReuseBlockIds);
 
   friend class internal::LogBlockContainer;
+  friend class internal::LogWritableBlock;
 
   // Type for the actual block map used to store all live blocks.
   // We use sparse_hash_map<> here to reduce memory overhead.

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/deltafile.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/deltafile.cc b/src/kudu/tablet/deltafile.cc
index 2db7811..b7db07f 100644
--- a/src/kudu/tablet/deltafile.cc
+++ b/src/kudu/tablet/deltafile.cc
@@ -79,8 +79,8 @@ using cfile::BlockPointer;
 using cfile::CFileReader;
 using cfile::IndexTreeIterator;
 using cfile::ReaderOptions;
+using fs::BlockTransaction;
 using fs::ReadableBlock;
-using fs::ScopedWritableBlockCloser;
 using fs::WritableBlock;
 
 namespace tablet {
@@ -113,16 +113,16 @@ Status DeltaFileWriter::Start() {
 }
 
 Status DeltaFileWriter::Finish() {
-  ScopedWritableBlockCloser closer;
-  RETURN_NOT_OK(FinishAndReleaseBlock(&closer));
-  return closer.CloseBlocks();
+  BlockTransaction transaction;
+  RETURN_NOT_OK(FinishAndReleaseBlock(&transaction));
+  return transaction.CommitCreatedBlocks();
 }
 
-Status DeltaFileWriter::FinishAndReleaseBlock(ScopedWritableBlockCloser* closer) {
+Status DeltaFileWriter::FinishAndReleaseBlock(BlockTransaction* transaction) {
   if (writer_->written_value_count() == 0) {
     return Status::Aborted("no deltas written");
   }
-  return writer_->FinishAndReleaseBlock(closer);
+  return writer_->FinishAndReleaseBlock(transaction);
 }
 
 Status DeltaFileWriter::DoAppendDelta(const DeltaKey &key,

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/deltafile.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/deltafile.h b/src/kudu/tablet/deltafile.h
index 43c5301..bd85ecb 100644
--- a/src/kudu/tablet/deltafile.h
+++ b/src/kudu/tablet/deltafile.h
@@ -64,8 +64,8 @@ struct ReaderOptions;
 } // namespace cfile
 
 namespace fs {
+class BlockTransaction;
 class ReadableBlock;
-class ScopedWritableBlockCloser;
 class WritableBlock;
 } // namespace fs
 
@@ -93,10 +93,12 @@ class DeltaFileWriter {
   // writer.
   Status Finish();
 
-  // Closes the delta file, releasing the underlying block to 'closer'.
+  // Closes the delta file, finalizing the underlying block and releasing
+  // it to 'transaction'.
+  //
   // Returns Status::Aborted() if no deltas were ever appended to this
   // writer.
-  Status FinishAndReleaseBlock(fs::ScopedWritableBlockCloser* closer);
+  Status FinishAndReleaseBlock(fs::BlockTransaction* transaction);
 
   // Append a given delta to the file. This must be called in ascending order
   // of (key, timestamp) for REDOS and ascending order of key, descending order

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/diskrowset.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/diskrowset.cc b/src/kudu/tablet/diskrowset.cc
index bf7caa1..aff5b8f 100644
--- a/src/kudu/tablet/diskrowset.cc
+++ b/src/kudu/tablet/diskrowset.cc
@@ -19,7 +19,6 @@
 
 #include <algorithm>
 #include <map>
-#include <mutex>
 #include <ostream>
 #include <vector>
 
@@ -82,8 +81,8 @@ class OpId;
 namespace tablet {
 
 using cfile::BloomFileWriter;
+using fs::BlockTransaction;
 using fs::CreateBlockOptions;
-using fs::ScopedWritableBlockCloser;
 using fs::WritableBlock;
 using log::LogAnchorRegistry;
 using std::shared_ptr;
@@ -228,12 +227,12 @@ Status DiskRowSetWriter::AppendBlock(const RowBlock &block) {
 
 Status DiskRowSetWriter::Finish() {
   TRACE_EVENT0("tablet", "DiskRowSetWriter::Finish");
-  ScopedWritableBlockCloser closer;
-  RETURN_NOT_OK(FinishAndReleaseBlocks(&closer));
-  return closer.CloseBlocks();
+  BlockTransaction transaction;
+  RETURN_NOT_OK(FinishAndReleaseBlocks(&transaction));
+  return transaction.CommitCreatedBlocks();
 }
 
-Status DiskRowSetWriter::FinishAndReleaseBlocks(ScopedWritableBlockCloser* closer) {
+Status DiskRowSetWriter::FinishAndReleaseBlocks(BlockTransaction* transaction) {
   TRACE_EVENT0("tablet", "DiskRowSetWriter::FinishAndReleaseBlocks");
   CHECK(!finished_);
 
@@ -254,7 +253,7 @@ Status DiskRowSetWriter::FinishAndReleaseBlocks(ScopedWritableBlockCloser* close
   key_index_writer()->AddMetadataPair(DiskRowSet::kMaxKeyMetaEntryName, last_enc_slice);
 
   // Finish writing the columns themselves.
-  RETURN_NOT_OK(col_writer_->FinishAndReleaseBlocks(closer));
+  RETURN_NOT_OK(col_writer_->FinishAndReleaseBlocks(transaction));
 
   // Put the column data blocks in the metadata.
   std::map<ColumnId, BlockId> flushed_blocks;
@@ -262,7 +261,7 @@ Status DiskRowSetWriter::FinishAndReleaseBlocks(ScopedWritableBlockCloser* close
   rowset_metadata_->SetColumnDataBlocks(flushed_blocks);
 
   if (ad_hoc_index_writer_ != nullptr) {
-    Status s = ad_hoc_index_writer_->FinishAndReleaseBlock(closer);
+    Status s = ad_hoc_index_writer_->FinishAndReleaseBlock(transaction);
     if (!s.ok()) {
       LOG(WARNING) << "Unable to Finish ad hoc index writer: " << s.ToString();
       return s;
@@ -270,7 +269,7 @@ Status DiskRowSetWriter::FinishAndReleaseBlocks(ScopedWritableBlockCloser* close
   }
 
   // Finish bloom.
-  Status s = bloom_writer_->FinishAndReleaseBlock(closer);
+  Status s = bloom_writer_->FinishAndReleaseBlock(transaction);
   if (!s.ok()) {
     LOG(WARNING) << "Unable to Finish bloom filter writer: " << s.ToString();
     return s;
@@ -421,7 +420,7 @@ Status RollingDiskRowSetWriter::FinishCurrentWriter() {
   }
   CHECK_EQ(state_, kStarted);
 
-  Status writer_status = cur_writer_->FinishAndReleaseBlocks(&block_closer_);
+  Status writer_status = cur_writer_->FinishAndReleaseBlocks(&block_transaction_);
 
   // If no rows were written (e.g. due to an empty flush or a compaction with all rows
   // deleted), FinishAndReleaseBlocks(...) returns Aborted. In that case, we don't
@@ -437,7 +436,7 @@ Status RollingDiskRowSetWriter::FinishCurrentWriter() {
 
     // Commit the UNDO block. Status::Aborted() indicates that there
     // were no UNDOs written.
-    Status s = cur_undo_writer_->FinishAndReleaseBlock(&block_closer_);
+    Status s = cur_undo_writer_->FinishAndReleaseBlock(&block_transaction_);
     if (!s.IsAborted()) {
       RETURN_NOT_OK(s);
       cur_drs_metadata_->CommitUndoDeltaDataBlock(cur_undo_ds_block_id_);
@@ -446,7 +445,7 @@ Status RollingDiskRowSetWriter::FinishCurrentWriter() {
     }
 
     // Same for the REDO block.
-    s = cur_redo_writer_->FinishAndReleaseBlock(&block_closer_);
+    s = cur_redo_writer_->FinishAndReleaseBlock(&block_transaction_);
     if (!s.IsAborted()) {
       RETURN_NOT_OK(s);
       cur_drs_metadata_->CommitRedoDeltaDataBlock(0, cur_redo_ds_block_id_);
@@ -473,7 +472,7 @@ Status RollingDiskRowSetWriter::Finish() {
   DCHECK_EQ(state_, kStarted);
 
   RETURN_NOT_OK(FinishCurrentWriter());
-  RETURN_NOT_OK(block_closer_.CloseBlocks());
+  RETURN_NOT_OK(block_transaction_.CommitCreatedBlocks());
 
   state_ = kFinished;
   return Status::OK();

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/diskrowset.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/diskrowset.h b/src/kudu/tablet/diskrowset.h
index 919e8fc..bdfff6c 100644
--- a/src/kudu/tablet/diskrowset.h
+++ b/src/kudu/tablet/diskrowset.h
@@ -105,9 +105,9 @@ class DiskRowSetWriter {
   // If no rows were written, returns Status::Aborted().
   Status Finish();
 
-  // Closes the CFiles, releasing the underlying blocks to 'closer'.
-  // If no rows were written, returns Status::Aborted().
-  Status FinishAndReleaseBlocks(fs::ScopedWritableBlockCloser* closer);
+  // Closes the CFiles, finalizing the underlying blocks and releasing
+  // them to 'transaction'. If no rows were written, returns Status::Aborted().
+  Status FinishAndReleaseBlocks(fs::BlockTransaction* transaction);
 
   // The base DiskRowSetWriter never rolls. This method is necessary for tests
   // which are templatized on the writer type.
@@ -267,9 +267,9 @@ class RollingDiskRowSetWriter {
   int64_t written_count_;
   uint64_t written_size_;
 
-  // Syncs and closes all outstanding blocks when the rolling writer is
-  // destroyed.
-  fs::ScopedWritableBlockCloser block_closer_;
+  // Syncs and commits all writes of outstanding blocks when the rolling
+  // writer is destroyed.
+  fs::BlockTransaction block_transaction_;
 
   DISALLOW_COPY_AND_ASSIGN(RollingDiskRowSetWriter);
 };

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/multi_column_writer.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/multi_column_writer.cc b/src/kudu/tablet/multi_column_writer.cc
index 56bfed1..43b5781 100644
--- a/src/kudu/tablet/multi_column_writer.cc
+++ b/src/kudu/tablet/multi_column_writer.cc
@@ -37,8 +37,8 @@ namespace kudu {
 namespace tablet {
 
 using cfile::CFileWriter;
+using fs::BlockTransaction;
 using fs::CreateBlockOptions;
-using fs::ScopedWritableBlockCloser;
 using fs::WritableBlock;
 using std::unique_ptr;
 
@@ -119,16 +119,16 @@ Status MultiColumnWriter::AppendBlock(const RowBlock& block) {
 }
 
 Status MultiColumnWriter::Finish() {
-  ScopedWritableBlockCloser closer;
-  RETURN_NOT_OK(FinishAndReleaseBlocks(&closer));
-  return closer.CloseBlocks();
+  BlockTransaction transaction;
+  RETURN_NOT_OK(FinishAndReleaseBlocks(&transaction));
+  return transaction.CommitCreatedBlocks();
 }
 
-Status MultiColumnWriter::FinishAndReleaseBlocks(ScopedWritableBlockCloser* closer) {
+Status MultiColumnWriter::FinishAndReleaseBlocks(BlockTransaction* transaction) {
   CHECK(!finished_);
   for (int i = 0; i < schema_->num_columns(); i++) {
     CFileWriter *writer = cfile_writers_[i];
-    Status s = writer->FinishAndReleaseBlock(closer);
+    Status s = writer->FinishAndReleaseBlock(transaction);
     if (!s.ok()) {
       LOG(WARNING) << "Unable to Finish writer for column " <<
         schema_->column(i).ToString() << ": " << s.ToString();

http://git-wip-us.apache.org/repos/asf/kudu/blob/59ca360b/src/kudu/tablet/multi_column_writer.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/multi_column_writer.h b/src/kudu/tablet/multi_column_writer.h
index 61fe007..7502d2f 100644
--- a/src/kudu/tablet/multi_column_writer.h
+++ b/src/kudu/tablet/multi_column_writer.h
@@ -40,7 +40,7 @@ class CFileWriter;
 } // namespace cfile
 
 namespace fs {
-class ScopedWritableBlockCloser;
+class BlockTransaction;
 } // namespace fs
 
 namespace tablet {
@@ -68,9 +68,9 @@ class MultiColumnWriter {
   // The file's blocks may be retrieved using FlushedBlocks().
   Status Finish();
 
-  // Close the in-progress CFiles, releasing the underlying writable blocks
-  // to 'closer'.
-  Status FinishAndReleaseBlocks(fs::ScopedWritableBlockCloser* closer);
+  // Close the in-progress CFiles, finalizing the underlying writable
+  // blocks and releasing them to 'transaction'.
+  Status FinishAndReleaseBlocks(fs::BlockTransaction* transaction);
 
   // Return the number of bytes written so far.
   size_t written_size() const;


[2/2] kudu git commit: KUDU-1726: Avoid fsync-per-block in tablet copy

Posted by da...@apache.org.
KUDU-1726: Avoid fsync-per-block in tablet copy

This patch incorporates BlockTransaction API into tablet
copy, to avoid fsync() on each block during copying. Blocks
are synced to disk together once the tablet copy is complete.
It also introduces a new block manager metric 'total_disk_sync'
to keep track of block data disk synchronization count.

I did a manual test to copy tablet with size of ~37GB from one
tablet server to another on el6 with ext4. Each tablet server
has its own dedicated single disk.
'kudu remote_replica copy c53ffbc2ede84b6d9af2da607024d131
localhost:3334 localhost:3335'

With this change, the operation time dropped down from ~718s
to ~523s.

Change-Id: I7534699f589c7060ffe32d7ac67546476cf21e76
Reviewed-on: http://gerrit.cloudera.org:8080/7701
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3d42a264
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3d42a264
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3d42a264

Branch: refs/heads/master
Commit: 3d42a264c15ded1001284a851b7f30a2b4465411
Parents: 59ca360
Author: hahao <ha...@cloudera.com>
Authored: Thu Aug 24 15:52:08 2017 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Fri Aug 25 02:16:43 2017 +0000

----------------------------------------------------------------------
 src/kudu/fs/block_manager_metrics.cc        |  8 +++++-
 src/kudu/fs/block_manager_metrics.h         |  1 +
 src/kudu/fs/file_block_manager.cc           |  2 ++
 src/kudu/fs/log_block_manager.cc            |  3 +++
 src/kudu/tserver/tablet_copy_client-test.cc | 32 ++++++++++++++++++++++--
 src/kudu/tserver/tablet_copy_client.cc      | 25 ++++++++++--------
 src/kudu/tserver/tablet_copy_client.h       | 17 ++++++++++---
 7 files changed, 71 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/fs/block_manager_metrics.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager_metrics.cc b/src/kudu/fs/block_manager_metrics.cc
index c53f26f..26bb50e 100644
--- a/src/kudu/fs/block_manager_metrics.cc
+++ b/src/kudu/fs/block_manager_metrics.cc
@@ -49,6 +49,11 @@ METRIC_DEFINE_counter(server, block_manager_total_bytes_read,
                       kudu::MetricUnit::kBytes,
                       "Number of bytes of block data read since service start");
 
+METRIC_DEFINE_counter(server, block_manager_total_disk_sync,
+                      "Block Data Disk Synchronization Count",
+                      kudu::MetricUnit::kBlocks,
+                      "Number of disk synchronizations of block data since service start");
+
 namespace kudu {
 namespace fs {
 namespace internal {
@@ -61,7 +66,8 @@ BlockManagerMetrics::BlockManagerMetrics(const scoped_refptr<MetricEntity>& enti
     MINIT(total_readable_blocks),
     MINIT(total_writable_blocks),
     MINIT(total_bytes_read),
-    MINIT(total_bytes_written) {
+    MINIT(total_bytes_written),
+    MINIT(total_disk_sync) {
 }
 #undef GINIT
 #undef MINIT

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/fs/block_manager_metrics.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager_metrics.h b/src/kudu/fs/block_manager_metrics.h
index 760f98b..03d9872 100644
--- a/src/kudu/fs/block_manager_metrics.h
+++ b/src/kudu/fs/block_manager_metrics.h
@@ -36,6 +36,7 @@ struct BlockManagerMetrics {
   scoped_refptr<Counter> total_writable_blocks;
   scoped_refptr<Counter> total_bytes_read;
   scoped_refptr<Counter> total_bytes_written;
+  scoped_refptr<Counter> total_disk_sync;
 };
 
 } // namespace internal

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/fs/file_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/file_block_manager.cc b/src/kudu/fs/file_block_manager.cc
index bd5d35d..dda2a31 100644
--- a/src/kudu/fs/file_block_manager.cc
+++ b/src/kudu/fs/file_block_manager.cc
@@ -390,6 +390,7 @@ Status FileWritableBlock::Close(SyncMode mode) {
     // Safer to synchronize data first, then metadata.
     VLOG(3) << "Syncing block " << id();
     if (FLAGS_enable_data_block_fsync) {
+      if (block_manager_->metrics_) block_manager_->metrics_->total_disk_sync->Increment();
       sync = writer_->Sync();
     }
     if (sync.ok()) {
@@ -559,6 +560,7 @@ Status FileBlockManager::SyncMetadata(const internal::FileBlockLocation& locatio
   // Sync them.
   if (FLAGS_enable_data_block_fsync) {
     for (const string& s : to_sync) {
+      if (metrics_) metrics_->total_disk_sync->Increment();
       RETURN_NOT_OK_HANDLE_DISK_FAILURE(env_->SyncDir(s),
           error_manager_->RunErrorNotificationCb(location.data_dir()));
     }

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index cf80d4e..2b06afa 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -991,6 +991,7 @@ Status LogBlockContainer::FlushMetadata() {
 Status LogBlockContainer::SyncData() {
   DCHECK(!read_only());
   if (FLAGS_enable_data_block_fsync) {
+    if (metrics_) metrics_->generic_metrics.total_disk_sync->Increment();
     RETURN_NOT_OK_HANDLE_ERROR(data_file_->Sync());
   }
   return Status::OK();
@@ -999,6 +1000,7 @@ Status LogBlockContainer::SyncData() {
 Status LogBlockContainer::SyncMetadata() {
   DCHECK(!read_only());
   if (FLAGS_enable_data_block_fsync) {
+    if (metrics_) metrics_->generic_metrics.total_disk_sync->Increment();
     RETURN_NOT_OK_HANDLE_ERROR(metadata_file_->Sync());
   }
   return Status::OK();
@@ -1886,6 +1888,7 @@ Status LogBlockManager::SyncContainer(const LogBlockContainer& container) {
   }
 
   if (to_sync && FLAGS_enable_data_block_fsync) {
+    if (metrics_) metrics_->generic_metrics.total_disk_sync->Increment();
     s = env_->SyncDir(container.data_dir()->dir());
 
     // If SyncDir fails, the container directory must be restored to

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/tserver/tablet_copy_client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client-test.cc b/src/kudu/tserver/tablet_copy_client-test.cc
index c6afa49..e299257 100644
--- a/src/kudu/tserver/tablet_copy_client-test.cc
+++ b/src/kudu/tserver/tablet_copy_client-test.cc
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+#include <gflags/gflags_declare.h>
 #include <glog/logging.h>
 #include <glog/stl_logging.h>
 #include <gtest/gtest.h>
@@ -38,6 +39,7 @@
 #include "kudu/fs/block_id.h"
 #include "kudu/fs/block_manager.h"
 #include "kudu/fs/fs_manager.h"
+#include "kudu/gutil/casts.h"
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/port.h"
 #include "kudu/gutil/ref_counted.h"
@@ -53,6 +55,7 @@
 #include "kudu/util/env.h"
 #include "kudu/util/env_util.h"
 #include "kudu/util/faststring.h"
+#include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/net/net_util.h"
 #include "kudu/util/slice.h"
@@ -63,6 +66,12 @@ using std::shared_ptr;
 using std::string;
 using std::vector;
 
+DECLARE_string(block_manager);
+
+METRIC_DECLARE_counter(block_manager_total_disk_sync);
+METRIC_DECLARE_counter(block_manager_total_writable_blocks);
+METRIC_DECLARE_gauge_uint64(log_block_manager_containers);
+
 namespace kudu {
 namespace tserver {
 
@@ -78,7 +87,13 @@ class TabletCopyClientTest : public TabletCopyTest {
   virtual void SetUp() OVERRIDE {
     NO_FATALS(TabletCopyTest::SetUp());
 
-    fs_manager_.reset(new FsManager(Env::Default(), GetTestPath("client_tablet")));
+    const string kTestDir = GetTestPath("client_tablet");
+    FsManagerOpts opts;
+    opts.wal_path = kTestDir;
+    opts.data_paths = { kTestDir };
+    metric_entity_ = METRIC_ENTITY_server.Instantiate(&metric_registry_, "test");
+    opts.metric_entity = metric_entity_;
+    fs_manager_.reset(new FsManager(Env::Default(), opts));
     ASSERT_OK(fs_manager_->CreateInitialFileSystemLayout());
     ASSERT_OK(fs_manager_->Open());
 
@@ -101,6 +116,8 @@ class TabletCopyClientTest : public TabletCopyTest {
  protected:
   Status CompareFileContents(const string& path1, const string& path2);
 
+  MetricRegistry metric_registry_;
+  scoped_refptr<MetricEntity> metric_entity_;
   gscoped_ptr<FsManager> fs_manager_;
   shared_ptr<rpc::Messenger> messenger_;
   gscoped_ptr<TabletCopyClient> client_;
@@ -160,7 +177,9 @@ TEST_F(TabletCopyClientTest, TestDownloadBlock) {
 
   // Check that the client downloaded the block and verification passed.
   BlockId new_block_id;
-  ASSERT_OK(client_->DownloadBlock(block_id, &new_block_id));
+  fs::BlockTransaction transaction;
+  ASSERT_OK(client_->DownloadBlock(block_id, &new_block_id, &transaction));
+  ASSERT_OK(transaction.CommitCreatedBlocks());
 
   // Ensure it placed the block where we expected it to.
   ASSERT_OK(ReadLocalBlockFile(fs_manager_.get(), new_block_id, &scratch, &slice));
@@ -226,6 +245,15 @@ TEST_F(TabletCopyClientTest, TestDownloadAllBlocks) {
   // Download all the blocks.
   ASSERT_OK(client_->DownloadBlocks());
 
+  // Verify the disk synchronization count.
+  if (FLAGS_block_manager == "log") {
+    ASSERT_EQ(3, down_cast<Counter*>(
+        metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
+  } else {
+    ASSERT_EQ(14, down_cast<Counter*>(
+        metric_entity_->FindOrNull(METRIC_block_manager_total_disk_sync).get())->value());
+  }
+
   // After downloading blocks, verify that the old and remote and local
   // superblock point to the same number of blocks.
   vector<BlockId> old_data_blocks = ListBlocks(*client_->remote_superblock_);

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/tserver/tablet_copy_client.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.cc b/src/kudu/tserver/tablet_copy_client.cc
index 07b20ff..2b894fe 100644
--- a/src/kudu/tserver/tablet_copy_client.cc
+++ b/src/kudu/tserver/tablet_copy_client.cc
@@ -91,6 +91,7 @@ using consensus::ConsensusMetadata;
 using consensus::ConsensusMetadataManager;
 using consensus::OpId;
 using env_util::CopyFile;
+using fs::BlockTransaction;
 using fs::CreateBlockOptions;
 using fs::WritableBlock;
 using rpc::Messenger;
@@ -458,6 +459,7 @@ Status TabletCopyClient::DownloadBlocks() {
   // as each block downloads.
   int block_count = 0;
   LOG_WITH_PREFIX(INFO) << "Starting download of " << num_remote_blocks << " data blocks...";
+  BlockTransaction transaction;
   for (const RowSetDataPB& src_rowset : remote_superblock_->rowsets()) {
     // Create rowset.
     RowSetDataPB* dst_rowset = superblock_->add_rowsets();
@@ -478,7 +480,7 @@ Status TabletCopyClient::DownloadBlocks() {
     for (const ColumnDataPB& src_col : src_rowset.columns()) {
       BlockIdPB new_block_id;
       RETURN_NOT_OK(DownloadAndRewriteBlock(src_col.block(), num_remote_blocks,
-                                            &block_count, &new_block_id));
+                                            &block_count, &new_block_id, &transaction));
       ColumnDataPB* dst_col = dst_rowset->add_columns();
       *dst_col = src_col;
       *dst_col->mutable_block() = new_block_id;
@@ -486,7 +488,7 @@ Status TabletCopyClient::DownloadBlocks() {
     for (const DeltaDataPB& src_redo : src_rowset.redo_deltas()) {
       BlockIdPB new_block_id;
       RETURN_NOT_OK(DownloadAndRewriteBlock(src_redo.block(), num_remote_blocks,
-                                            &block_count, &new_block_id));
+                                            &block_count, &new_block_id, &transaction));
       DeltaDataPB* dst_redo = dst_rowset->add_redo_deltas();
       *dst_redo = src_redo;
       *dst_redo->mutable_block() = new_block_id;
@@ -494,7 +496,7 @@ Status TabletCopyClient::DownloadBlocks() {
     for (const DeltaDataPB& src_undo : src_rowset.undo_deltas()) {
       BlockIdPB new_block_id;
       RETURN_NOT_OK(DownloadAndRewriteBlock(src_undo.block(), num_remote_blocks,
-                                            &block_count, &new_block_id));
+                                            &block_count, &new_block_id, &transaction));
       DeltaDataPB* dst_undo = dst_rowset->add_undo_deltas();
       *dst_undo = src_undo;
       *dst_undo->mutable_block() = new_block_id;
@@ -502,18 +504,18 @@ Status TabletCopyClient::DownloadBlocks() {
     if (src_rowset.has_bloom_block()) {
       BlockIdPB new_block_id;
       RETURN_NOT_OK(DownloadAndRewriteBlock(src_rowset.bloom_block(), num_remote_blocks,
-                                            &block_count, &new_block_id));
+                                            &block_count, &new_block_id, &transaction));
       *dst_rowset->mutable_bloom_block() = new_block_id;
     }
     if (src_rowset.has_adhoc_index_block()) {
       BlockIdPB new_block_id;
       RETURN_NOT_OK(DownloadAndRewriteBlock(src_rowset.adhoc_index_block(), num_remote_blocks,
-                                            &block_count, &new_block_id));
+                                            &block_count, &new_block_id, &transaction));
       *dst_rowset->mutable_adhoc_index_block() = new_block_id;
     }
   }
 
-  return Status::OK();
+  return transaction.CommitCreatedBlocks();
 }
 
 Status TabletCopyClient::DownloadWAL(uint64_t wal_segment_seqno) {
@@ -563,13 +565,14 @@ Status TabletCopyClient::WriteConsensusMetadata() {
 Status TabletCopyClient::DownloadAndRewriteBlock(const BlockIdPB& src_block_id,
                                                  int num_blocks,
                                                  int* block_count,
-                                                 BlockIdPB* dest_block_id) {
+                                                 BlockIdPB* dest_block_id,
+                                                 BlockTransaction* transaction) {
   BlockId old_block_id(BlockId::FromPB(src_block_id));
   SetStatusMessage(Substitute("Downloading block $0 ($1/$2)",
                               old_block_id.ToString(),
                               *block_count + 1, num_blocks));
   BlockId new_block_id;
-  RETURN_NOT_OK_PREPEND(DownloadBlock(old_block_id, &new_block_id),
+  RETURN_NOT_OK_PREPEND(DownloadBlock(old_block_id, &new_block_id, transaction),
       "Unable to download block with id " + old_block_id.ToString());
 
   new_block_id.CopyToPB(dest_block_id);
@@ -578,7 +581,8 @@ Status TabletCopyClient::DownloadAndRewriteBlock(const BlockIdPB& src_block_id,
 }
 
 Status TabletCopyClient::DownloadBlock(const BlockId& old_block_id,
-                                       BlockId* new_block_id) {
+                                       BlockId* new_block_id,
+                                       BlockTransaction* transaction) {
   VLOG_WITH_PREFIX(1) << "Downloading block with block_id " << old_block_id.ToString();
 
   unique_ptr<WritableBlock> block;
@@ -593,7 +597,8 @@ Status TabletCopyClient::DownloadBlock(const BlockId& old_block_id,
                                    old_block_id.ToString()));
 
   *new_block_id = block->id();
-  RETURN_NOT_OK_PREPEND(block->Close(), "Unable to close block");
+  RETURN_NOT_OK_PREPEND(block->Finalize(), "Unable to finalize block");
+  transaction->AddCreatedBlock(std::move(block));
   return Status::OK();
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/3d42a264/src/kudu/tserver/tablet_copy_client.h
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_copy_client.h b/src/kudu/tserver/tablet_copy_client.h
index 429c05b..f8a47bc 100644
--- a/src/kudu/tserver/tablet_copy_client.h
+++ b/src/kudu/tserver/tablet_copy_client.h
@@ -41,6 +41,10 @@ class ConsensusMetadataManager;
 class ConsensusStatePB;
 } // namespace consensus
 
+namespace fs {
+class BlockTransaction;
+} // namespace fs
+
 namespace rpc {
 class Messenger;
 class RpcController;
@@ -158,7 +162,8 @@ class TabletCopyClient {
 
   // Download the remote block specified by 'src_block_id'. 'num_blocks' should
   // be given as the total number of blocks there are to download (for logging
-  // purposes).
+  // purposes). Add the block to the given transaction, to close blocks belonging
+  // to a transaction together when the copying is complete.
   //
   // On success:
   // - 'dest_block_id' is set to the new ID of the downloaded block.
@@ -166,13 +171,17 @@ class TabletCopyClient {
   Status DownloadAndRewriteBlock(const BlockIdPB& src_block_id,
                                  int num_blocks,
                                  int* block_count,
-                                 BlockIdPB* dest_block_id);
+                                 BlockIdPB* dest_block_id,
+                                 fs::BlockTransaction* transaction);
 
   // Download a single block.
-  // Data block is opened with options so that it will fsync() on close.
+  // Data block is opened with new ID. After downloading, the block is finalized
+  // and added to the given transaction.
   //
   // On success, 'new_block_id' is set to the new ID of the downloaded block.
-  Status DownloadBlock(const BlockId& old_block_id, BlockId* new_block_id);
+  Status DownloadBlock(const BlockId& old_block_id,
+                       BlockId* new_block_id,
+                       fs::BlockTransaction* transaction);
 
   // Download a single remote file. The block and WAL implementations delegate
   // to this method when downloading files.