You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/05/17 01:18:48 UTC

incubator-kudu git commit: KUDU-1131. Avoid CHECK failure during compaction when an operation is slow to commit

Repository: incubator-kudu
Updated Branches:
  refs/heads/master c3553ac5b -> 3512b1ac6


KUDU-1131. Avoid CHECK failure during compaction when an operation is slow to commit

This fixes a CHECK failure in the following case:

- an operation with txid 5 is replicated, but not yet applying
- an operation with txid 6 is replicated, and starts to apply
- the tablet flushes. We wait for txid 6 to commit, since it was already
  applying when the flush snapshot was taken.
-- the resulting UNDO file now includes an UNDO at txid 6, so its
   max_timestamp is 6
- the tablet issues a compaction, before txid 5 has committed

This would trigger a CHECK because we see that the current snapshot has txid 5
as uncommitted, but there is an UNDO delta with a max txid of 6. With the code
before this patch, that would have erroneously made the compaction code think
the UNDO file was actually a REDO file, since its time range overlapped the
current snapshot.

To fix this, I added a new call to DeltaTracker to specifically fetch UNDO or
REDO delta files, rather than relying on the time range and snapshot to do so.
The CHECK can now safely be removed.

The original commit that added this CHECK was 1a6b80a310a7de3519d78a2f5e90ecaae1cf405a.
The commit message there mentions that linked_list-test was modified at that point
to act as a regression test for the original bug. I ran that test 500 times and
they all passed:
http://dist-test.cloudera.org//job?job_id=todd.1463187608.28912

I also ran mt-tablet-test "DoTestAllAtOnce" 4000 times. This test was flaky (1
or 2 failures out of 4000) prior to this change and now passed:
http://dist-test.cloudera.org/job?job_id=todd.1463186765.27185

Change-Id: Ie16f2c6d190a322c107d60312d4c35d7aa409c43
Reviewed-on: http://gerrit.cloudera.org:8080/3073
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/3512b1ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/3512b1ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/3512b1ac

Branch: refs/heads/master
Commit: 3512b1ac68f92ebe1e4e632d9c9dd11396edf1b3
Parents: c3553ac
Author: Todd Lipcon <to...@apache.org>
Authored: Fri May 13 17:51:34 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue May 17 01:18:30 2016 +0000

----------------------------------------------------------------------
 src/kudu/tablet/compaction.cc    | 27 +++++++++----------------
 src/kudu/tablet/delta_tracker.cc | 38 ++++++++++-------------------------
 src/kudu/tablet/delta_tracker.h  | 38 ++++++++++++++++++++++++-----------
 3 files changed, 46 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/3512b1ac/src/kudu/tablet/compaction.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/compaction.cc b/src/kudu/tablet/compaction.cc
index 8648106..af349f7 100644
--- a/src/kudu/tablet/compaction.cc
+++ b/src/kudu/tablet/compaction.cc
@@ -523,28 +523,19 @@ Status CompactionInput::Create(const DiskRowSet &rowset,
                                gscoped_ptr<CompactionInput>* out) {
   CHECK(projection->has_column_ids());
 
-  // Assertion which checks for an earlier bug where the compaction snapshot
-  // chosen was too early. This resulted in UNDO files being mistakenly
-  // identified as REDO files and corruption ensued. If the assertion fails,
-  // the process crashes; only unrelated I/O-related errors are returned.
-  RETURN_NOT_OK_PREPEND(rowset.delta_tracker_->CheckSnapshotComesAfterAllUndos(snap),
-                        "Could not open UNDOs");
-
   shared_ptr<ColumnwiseIterator> base_cwise(rowset.base_data_->NewIterator(projection));
   gscoped_ptr<RowwiseIterator> base_iter(new MaterializingIterator(base_cwise));
-  // Creates a DeltaIteratorMerger that will only include part of the redo deltas,
-  // since 'snap' will be after the snapshot of the last flush/compaction,
-  // i.e. past all undo deltas's max transaction ID.
+
+  // Creates a DeltaIteratorMerger that will only include the relevant REDO deltas.
   shared_ptr<DeltaIterator> redo_deltas;
-  RETURN_NOT_OK_PREPEND(rowset.delta_tracker_->NewDeltaIterator(projection, snap, &redo_deltas),
-                        "Could not open REDOs");
-  // Creates a DeltaIteratorMerger that will only include undo deltas, since
-  // MvccSnapshot::CreateSnapshotIncludingNoTransactions() excludes all redo
-  // deltas's min transaction ID.
+  RETURN_NOT_OK_PREPEND(rowset.delta_tracker_->NewDeltaIterator(
+      projection, snap, DeltaTracker::REDOS_ONLY, &redo_deltas), "Could not open REDOs");
+  // Creates a DeltaIteratorMerger that will only include UNDO deltas. Using the
+  // "empty" snapshot ensures that all deltas are included.
   shared_ptr<DeltaIterator> undo_deltas;
-  RETURN_NOT_OK_PREPEND(rowset.delta_tracker_->NewDeltaIterator(projection,
-          MvccSnapshot::CreateSnapshotIncludingNoTransactions(),
-          &undo_deltas), "Could not open UNDOs");
+  RETURN_NOT_OK_PREPEND(rowset.delta_tracker_->NewDeltaIterator(
+      projection, MvccSnapshot::CreateSnapshotIncludingNoTransactions(),
+      DeltaTracker::UNDOS_ONLY, &undo_deltas), "Could not open UNDOs");
 
   out->reset(new DiskRowSetCompactionInput(std::move(base_iter), redo_deltas, undo_deltas));
   return Status::OK();

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/3512b1ac/src/kudu/tablet/delta_tracker.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/delta_tracker.cc b/src/kudu/tablet/delta_tracker.cc
index 13dfbd4..0b940d9 100644
--- a/src/kudu/tablet/delta_tracker.cc
+++ b/src/kudu/tablet/delta_tracker.cc
@@ -269,41 +269,24 @@ Status DeltaTracker::DoCompactStores(size_t start_idx, size_t end_idx,
   return Status::OK();
 }
 
-void DeltaTracker::CollectStores(vector<shared_ptr<DeltaStore> > *deltas) const {
+void DeltaTracker::CollectStores(vector<shared_ptr<DeltaStore>>* deltas,
+                                 WhichStores which) const {
   lock_guard<rw_spinlock> lock(&component_lock_);
-  deltas->assign(undo_delta_stores_.begin(), undo_delta_stores_.end());
-  deltas->insert(deltas->end(), redo_delta_stores_.begin(), redo_delta_stores_.end());
-  deltas->push_back(dms_);
-}
-
-Status DeltaTracker::CheckSnapshotComesAfterAllUndos(const MvccSnapshot& snap) const {
-  std::vector<shared_ptr<DeltaStore> > undos;
-  {
-    lock_guard<rw_spinlock> lock(&component_lock_);
-    undos = undo_delta_stores_;
+  if (which != REDOS_ONLY) {
+    deltas->assign(undo_delta_stores_.begin(), undo_delta_stores_.end());
   }
-  for (const shared_ptr<DeltaStore>& undo : undos) {
-    DeltaFileReader* dfr = down_cast<DeltaFileReader*>(undo.get());
-
-    // Even though IsRelevantForSnapshot() is safe to call without
-    // initializing the reader, the assertion being tested by this function
-    // will probably fail without real delta stats.
-    RETURN_NOT_OK(dfr->Init());
-
-    CHECK(!dfr->IsRelevantForSnapshot(snap))
-      << "Invalid snapshot " << snap.ToString()
-      << " does not come after undo file " << undo->ToString()
-      << " with stats: " << dfr->delta_stats().ToString();
+  if (which != UNDOS_ONLY) {
+    deltas->insert(deltas->end(), redo_delta_stores_.begin(), redo_delta_stores_.end());
+    deltas->push_back(dms_);
   }
-
-  return Status::OK();
 }
 
 Status DeltaTracker::NewDeltaIterator(const Schema* schema,
                                       const MvccSnapshot& snap,
+                                      WhichStores which,
                                       shared_ptr<DeltaIterator>* out) const {
   std::vector<shared_ptr<DeltaStore> > stores;
-  CollectStores(&stores);
+  CollectStores(&stores, which);
   return DeltaIteratorMerger::Create(stores, schema, snap, out);
 }
 
@@ -411,7 +394,8 @@ Status DeltaTracker::FlushDMS(DeltaMemStore* dms,
   gscoped_ptr<DeltaStats> stats;
   RETURN_NOT_OK(dms->FlushToFile(&dfw, &stats));
   RETURN_NOT_OK(dfw.Finish());
-  LOG(INFO) << "Flushed delta block: " << block_id.ToString();
+  LOG(INFO) << "Flushed delta block: " << block_id.ToString()
+            << " ts range: [" << stats->min_timestamp() << ", " << stats->max_timestamp() << "]";
 
   // Now re-open for read
   gscoped_ptr<ReadableBlock> readable_block;

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/3512b1ac/src/kudu/tablet/delta_tracker.h
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/delta_tracker.h b/src/kudu/tablet/delta_tracker.h
index de22d1d..d4e571b 100644
--- a/src/kudu/tablet/delta_tracker.h
+++ b/src/kudu/tablet/delta_tracker.h
@@ -76,15 +76,36 @@ class DeltaTracker {
                       const MvccSnapshot &mvcc_snap,
                       gscoped_ptr<ColumnwiseIterator>* out) const;
 
-  // TODO: this shouldn't need to return a shared_ptr, but there is some messiness
-  // where this has bled around.
+  // Enum used for NewDeltaIterator() and CollectStores() below.
+  // Determines whether all types of stores should be considered,
+  // or just UNDO or REDO stores.
+  enum WhichStores {
+    UNDOS_AND_REDOS,
+    UNDOS_ONLY,
+    REDOS_ONLY
+  };
+
+  // Create a new DeltaIterator which merges the delta stores tracked
+  // by this DeltaTracker. Depending on the value of 'which' (see above),
+  // this iterator may include UNDOs, REDOs, or both.
   //
   // 'schema' is the schema of the rows that are being read by the client.
   // It must remain valid for the lifetime of the returned iterator.
+  //
+  // TODO: this shouldn't need to return a shared_ptr, but there is some messiness
+  // where this has bled around.
   Status NewDeltaIterator(const Schema* schema,
                           const MvccSnapshot& snap,
+                          WhichStores which,
                           std::shared_ptr<DeltaIterator>* out) const;
 
+  Status NewDeltaIterator(const Schema* schema,
+                          const MvccSnapshot& snap,
+                          std::shared_ptr<DeltaIterator>* out) const {
+    return NewDeltaIterator(schema, snap, UNDOS_AND_REDOS, out);
+  }
+
+
   // Like NewDeltaIterator() but only includes file based stores, does not include
   // the DMS.
   // Returns the delta stores being merged in *included_stores.
@@ -95,14 +116,6 @@ class DeltaTracker {
     std::vector<std::shared_ptr<DeltaStore> >* included_stores,
     std::shared_ptr<DeltaIterator>* out) const;
 
-  // CHECKs that the given snapshot includes all of the UNDO stores in this
-  // delta tracker. If this is not the case, crashes the process. This is
-  // used as an assertion during compaction, where we always expect the
-  // compaction snapshot to be in the future relative to any UNDOs.
-  //
-  // Returns a bad status in the event of an I/O related error.
-  Status CheckSnapshotComesAfterAllUndos(const MvccSnapshot& snap) const;
-
   Status Open();
 
   // Flushes the current DeltaMemStore and replaces it with a new one.
@@ -197,8 +210,9 @@ class DeltaTracker {
                   std::shared_ptr<DeltaFileReader>* dfr,
                   MetadataFlushType flush_type);
 
-  // This collects all undo and redo stores.
-  void CollectStores(vector<std::shared_ptr<DeltaStore> > *stores) const;
+  // This collects undo and/or redo stores into '*stores'.
+  void CollectStores(vector<std::shared_ptr<DeltaStore>>* stores,
+                     WhichStores which) const;
 
   // Performs the actual compaction. Results of compaction are written to "block",
   // while delta stores that underwent compaction are appended to "compacted_stores", while