You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/12/06 23:50:45 UTC

kudu git commit: KUDU-1791: tighten up read only usage in log block manager

Repository: kudu
Updated Branches:
  refs/heads/master aaca775da -> 2453a6731


KUDU-1791: tighten up read only usage in log block manager

Over time we had introduced a couple new file deletion/truncation sites that
weren't checking read_only_ before making destructive changes.

This was originally caught by open-fs-readonly-itest. Though exceptionally
rare, it's difficult to conceive of a different test that would surface the
issue more regularly.

Change-Id: I029527b78148d68a002a3c42af5a52f179089a95
Reviewed-on: http://gerrit.cloudera.org:8080/5383
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2453a673
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2453a673
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2453a673

Branch: refs/heads/master
Commit: 2453a67310f62c01216e5a0ed08f192a08adc005
Parents: aaca775
Author: Adar Dembo <ad...@cloudera.com>
Authored: Tue Dec 6 10:20:43 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Tue Dec 6 23:49:49 2016 +0000

----------------------------------------------------------------------
 src/kudu/fs/block_manager-test.cc |  4 +--
 src/kudu/fs/log_block_manager.cc  | 63 +++++++++++++++++++++-------------
 2 files changed, 41 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/2453a673/src/kudu/fs/block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager-test.cc b/src/kudu/fs/block_manager-test.cc
index a854717..5209156 100644
--- a/src/kudu/fs/block_manager-test.cc
+++ b/src/kudu/fs/block_manager-test.cc
@@ -102,8 +102,8 @@ class BlockManagerTest : public KuduTest {
 
  protected:
   T* CreateBlockManager(const scoped_refptr<MetricEntity>& metric_entity,
-                                   const shared_ptr<MemTracker>& parent_mem_tracker,
-                                   const vector<string>& paths) {
+                        const shared_ptr<MemTracker>& parent_mem_tracker,
+                        const vector<string>& paths) {
     BlockManagerOptions opts;
     opts.metric_entity = metric_entity;
     opts.parent_mem_tracker = parent_mem_tracker;

http://git-wip-us.apache.org/repos/asf/kudu/blob/2453a673/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index cce6d84..52fd9c5 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -407,12 +407,19 @@ Status LogBlockContainer::Open(LogBlockManager* block_manager,
       RETURN_NOT_OK_PREPEND(s, "unable to determine data file size");
     }
 
-    if (metadata_size < pb_util::kPBContainerMinimumValidLength && data_size == 0) {
-      LOG(WARNING) << "Data and metadata files for container  " << common_path
-                   << " are both missing or empty. Deleting container.";
-      IgnoreResult(env->DeleteFile(metadata_path));
-      IgnoreResult(env->DeleteFile(data_path));
-      return Status::Aborted("orphaned empty metadata and data files removed");
+    if (metadata_size < pb_util::kPBContainerMinimumValidLength &&
+        data_size == 0) {
+      bool ro = block_manager->read_only_;
+      LOG(WARNING) << Substitute(
+          "Data and metadata files for container $0 are both missing or empty: $1",
+          common_path, ro ? "ignoring" : "deleting container");
+      if (!ro) {
+        IgnoreResult(env->DeleteFile(metadata_path));
+        IgnoreResult(env->DeleteFile(data_path));
+      }
+      return Status::Aborted(Substitute(
+          "orphaned empty metadata and data files $0",
+          ro ? "ignored" : "removed"));
     }
   }
 
@@ -498,19 +505,25 @@ Status LogBlockContainer::ReadContainerRecords(deque<BlockRecordPB>* records) co
     // We found a partial trailing record in a version of the pb container file
     // format that can reliably detect this. Consider this a failed partial
     // write and truncate the metadata file to remove this partial record.
-    uint64_t truncate_offset = pb_reader.offset();
-    LOG(WARNING) << "Log block manager: Found partial trailing metadata record in container "
-                  << ToString() << ": "
-                  << "Truncating metadata file to last valid offset: " << truncate_offset;
-    unique_ptr<RWFile> file;
-    RWFileOptions opts;
-    opts.mode = Env::OPEN_EXISTING;
-    RETURN_NOT_OK(block_manager_->env()->NewRWFile(opts, metadata_path, &file));
-    RETURN_NOT_OK(file->Truncate(truncate_offset));
-    RETURN_NOT_OK(file->Close());
-    // Reopen the PB writer so that it will refresh its metadata about the
-    // underlying file and resume appending to the new end of the file.
-    RETURN_NOT_OK(metadata_file_->Reopen());
+    LOG(WARNING) << Substitute(
+        "Log block manager: Found partial trailing metadata record in "
+        "container $0: $1",
+        ToString(), block_manager()->read_only_ ?
+            "ignoring" :
+            Substitute("truncating metadata file to last valid offset: $0",
+                       pb_reader.offset()));
+    if (!block_manager()->read_only_) {
+      unique_ptr<RWFile> file;
+      RWFileOptions opts;
+      opts.mode = Env::OPEN_EXISTING;
+      RETURN_NOT_OK(block_manager_->env()->NewRWFile(opts, metadata_path, &file));
+      RETURN_NOT_OK(file->Truncate(pb_reader.offset()));
+      RETURN_NOT_OK(file->Close());
+
+      // Reopen the PB writer so that it will refresh its metadata about the
+      // underlying file and resume appending to the new end of the file.
+      RETURN_NOT_OK(metadata_file_->Reopen());
+    }
     records->swap(local_records);
     return Status::OK();
   }
@@ -1511,11 +1524,13 @@ void LogBlockManager::OpenDataDir(DataDir* dir,
     // preallocated space off of the end of the container. This is a no-op for
     // non-full containers, where excess preallocated space is expected to be
     // (eventually) used.
-    s = container->TruncateDataToTotalBytesWritten();
-    if (!s.ok()) {
-      *result_status = s.CloneAndPrepend(Substitute(
-          "Could not truncate container $0", container->ToString()));
-      return;
+    if (!read_only_) {
+      s = container->TruncateDataToTotalBytesWritten();
+      if (!s.ok()) {
+        *result_status = s.CloneAndPrepend(Substitute(
+            "Could not truncate container $0", container->ToString()));
+        return;
+      }
     }
 
     next_block_id_.StoreMax(max_block_id + 1);