You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/12/06 23:50:45 UTC
kudu git commit: KUDU-1791: tighten up read only usage in log block
manager
Repository: kudu
Updated Branches:
refs/heads/master aaca775da -> 2453a6731
KUDU-1791: tighten up read only usage in log block manager
Over time we had introduced a couple new file deletion/truncation sites that
weren't checking read_only_ before making destructive changes.
This was originally caught by open-fs-readonly-itest. Though exceptionally
rare, it's difficult to conceive of a different test that would surface the
issue more regularly.
Change-Id: I029527b78148d68a002a3c42af5a52f179089a95
Reviewed-on: http://gerrit.cloudera.org:8080/5383
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2453a673
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2453a673
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2453a673
Branch: refs/heads/master
Commit: 2453a67310f62c01216e5a0ed08f192a08adc005
Parents: aaca775
Author: Adar Dembo <ad...@cloudera.com>
Authored: Tue Dec 6 10:20:43 2016 -0800
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Tue Dec 6 23:49:49 2016 +0000
----------------------------------------------------------------------
src/kudu/fs/block_manager-test.cc | 4 +--
src/kudu/fs/log_block_manager.cc | 63 +++++++++++++++++++++-------------
2 files changed, 41 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/2453a673/src/kudu/fs/block_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/block_manager-test.cc b/src/kudu/fs/block_manager-test.cc
index a854717..5209156 100644
--- a/src/kudu/fs/block_manager-test.cc
+++ b/src/kudu/fs/block_manager-test.cc
@@ -102,8 +102,8 @@ class BlockManagerTest : public KuduTest {
protected:
T* CreateBlockManager(const scoped_refptr<MetricEntity>& metric_entity,
- const shared_ptr<MemTracker>& parent_mem_tracker,
- const vector<string>& paths) {
+ const shared_ptr<MemTracker>& parent_mem_tracker,
+ const vector<string>& paths) {
BlockManagerOptions opts;
opts.metric_entity = metric_entity;
opts.parent_mem_tracker = parent_mem_tracker;
http://git-wip-us.apache.org/repos/asf/kudu/blob/2453a673/src/kudu/fs/log_block_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc
index cce6d84..52fd9c5 100644
--- a/src/kudu/fs/log_block_manager.cc
+++ b/src/kudu/fs/log_block_manager.cc
@@ -407,12 +407,19 @@ Status LogBlockContainer::Open(LogBlockManager* block_manager,
RETURN_NOT_OK_PREPEND(s, "unable to determine data file size");
}
- if (metadata_size < pb_util::kPBContainerMinimumValidLength && data_size == 0) {
- LOG(WARNING) << "Data and metadata files for container " << common_path
- << " are both missing or empty. Deleting container.";
- IgnoreResult(env->DeleteFile(metadata_path));
- IgnoreResult(env->DeleteFile(data_path));
- return Status::Aborted("orphaned empty metadata and data files removed");
+ if (metadata_size < pb_util::kPBContainerMinimumValidLength &&
+ data_size == 0) {
+ bool ro = block_manager->read_only_;
+ LOG(WARNING) << Substitute(
+ "Data and metadata files for container $0 are both missing or empty: $1",
+ common_path, ro ? "ignoring" : "deleting container");
+ if (!ro) {
+ IgnoreResult(env->DeleteFile(metadata_path));
+ IgnoreResult(env->DeleteFile(data_path));
+ }
+ return Status::Aborted(Substitute(
+ "orphaned empty metadata and data files $0",
+ ro ? "ignored" : "removed"));
}
}
@@ -498,19 +505,25 @@ Status LogBlockContainer::ReadContainerRecords(deque<BlockRecordPB>* records) co
// We found a partial trailing record in a version of the pb container file
// format that can reliably detect this. Consider this a failed partial
// write and truncate the metadata file to remove this partial record.
- uint64_t truncate_offset = pb_reader.offset();
- LOG(WARNING) << "Log block manager: Found partial trailing metadata record in container "
- << ToString() << ": "
- << "Truncating metadata file to last valid offset: " << truncate_offset;
- unique_ptr<RWFile> file;
- RWFileOptions opts;
- opts.mode = Env::OPEN_EXISTING;
- RETURN_NOT_OK(block_manager_->env()->NewRWFile(opts, metadata_path, &file));
- RETURN_NOT_OK(file->Truncate(truncate_offset));
- RETURN_NOT_OK(file->Close());
- // Reopen the PB writer so that it will refresh its metadata about the
- // underlying file and resume appending to the new end of the file.
- RETURN_NOT_OK(metadata_file_->Reopen());
+ LOG(WARNING) << Substitute(
+ "Log block manager: Found partial trailing metadata record in "
+ "container $0: $1",
+ ToString(), block_manager()->read_only_ ?
+ "ignoring" :
+ Substitute("truncating metadata file to last valid offset: $0",
+ pb_reader.offset()));
+ if (!block_manager()->read_only_) {
+ unique_ptr<RWFile> file;
+ RWFileOptions opts;
+ opts.mode = Env::OPEN_EXISTING;
+ RETURN_NOT_OK(block_manager_->env()->NewRWFile(opts, metadata_path, &file));
+ RETURN_NOT_OK(file->Truncate(pb_reader.offset()));
+ RETURN_NOT_OK(file->Close());
+
+ // Reopen the PB writer so that it will refresh its metadata about the
+ // underlying file and resume appending to the new end of the file.
+ RETURN_NOT_OK(metadata_file_->Reopen());
+ }
records->swap(local_records);
return Status::OK();
}
@@ -1511,11 +1524,13 @@ void LogBlockManager::OpenDataDir(DataDir* dir,
// preallocated space off of the end of the container. This is a no-op for
// non-full containers, where excess preallocated space is expected to be
// (eventually) used.
- s = container->TruncateDataToTotalBytesWritten();
- if (!s.ok()) {
- *result_status = s.CloneAndPrepend(Substitute(
- "Could not truncate container $0", container->ToString()));
- return;
+ if (!read_only_) {
+ s = container->TruncateDataToTotalBytesWritten();
+ if (!s.ok()) {
+ *result_status = s.CloneAndPrepend(Substitute(
+ "Could not truncate container $0", container->ToString()));
+ return;
+ }
}
next_block_id_.StoreMax(max_block_id + 1);