You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mp...@apache.org on 2016/08/24 18:48:15 UTC
[1/2] kudu git commit: Various comment / doc improvements
Repository: kudu
Updated Branches:
refs/heads/master cbc865108 -> df3d070e5
Various comment / doc improvements
* Update link to code review for supporting reinsert
* Fix comment in HybridClock header docs
* Improve comment documentation in compaction code
Change-Id: I45d45d265652b0c462aec1bedbcb4b254a2a05da
Reviewed-on: http://gerrit.cloudera.org:8080/3977
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/53507024
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/53507024
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/53507024
Branch: refs/heads/master
Commit: 535070248eda5ca1483546197ff7d935ff1bd5a4
Parents: cbc8651
Author: Mike Percy <mp...@apache.org>
Authored: Sun Aug 14 20:11:00 2016 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Wed Aug 24 00:54:08 2016 +0000
----------------------------------------------------------------------
src/kudu/common/row_changelist.h | 2 +-
src/kudu/server/hybrid_clock.h | 2 +-
src/kudu/tablet/delta_compaction.cc | 17 ++++++++++-------
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/53507024/src/kudu/common/row_changelist.h
----------------------------------------------------------------------
diff --git a/src/kudu/common/row_changelist.h b/src/kudu/common/row_changelist.h
index c48af75..480c374 100644
--- a/src/kudu/common/row_changelist.h
+++ b/src/kudu/common/row_changelist.h
@@ -56,7 +56,7 @@ class Schema;
// If type == kDelete, then no further data follows. The row is deleted.
//
// If type == kReinsert, then a "tuple-format" row follows. TODO: this will
-// be changed by http://gerrit.sjc.cloudera.com:8080/#/c/6318/ in the near future.
+// eventually be changed by something like https://gerrit.cloudera.org/975
//
// If type == kUpdate, then a sequence of column updates follow. Each update
// has the format:
http://git-wip-us.apache.org/repos/asf/kudu/blob/53507024/src/kudu/server/hybrid_clock.h
----------------------------------------------------------------------
diff --git a/src/kudu/server/hybrid_clock.h b/src/kudu/server/hybrid_clock.h
index bbf2fcb..1110f9b 100644
--- a/src/kudu/server/hybrid_clock.h
+++ b/src/kudu/server/hybrid_clock.h
@@ -145,7 +145,7 @@ class HybridClock : public Clock {
static std::string StringifyTimestamp(const Timestamp& timestamp);
// Sets the time to be returned by a mock call to the system clock, for tests.
- // Requires that 'FLAGS_use_mock_wall_clock' is set to true and that 'now_usec' is less
+ // Requires that 'FLAGS_use_mock_wall_clock' is set to true and that 'now_usec' is higher
// than the previously set time.
// NOTE: This refers to the time returned by the system clock, not the time returned
// by HybridClock, i.e. 'now_usec' is not a HybridTime timestmap and shouldn't have
http://git-wip-us.apache.org/repos/asf/kudu/blob/53507024/src/kudu/tablet/delta_compaction.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/delta_compaction.cc b/src/kudu/tablet/delta_compaction.cc
index 1cd7297..62bca8e 100644
--- a/src/kudu/tablet/delta_compaction.cc
+++ b/src/kudu/tablet/delta_compaction.cc
@@ -127,8 +127,8 @@ Status MajorDeltaCompaction::FlushRowSetAndDeltas() {
RETURN_NOT_OK(delta_iter_->PrepareBatch(n, DeltaIterator::PREPARE_FOR_COLLECT));
RETURN_NOT_OK(delta_iter_->CollectMutations(&redo_mutation_block, block.arena()));
- // 3) Apply new UNDO mutations for the current block. The REDO mutations are picked up
- // at step 6).
+ // 3) Write new UNDO mutations for the current block. The REDO mutations
+ // are written out in step 6.
vector<CompactionInputRow> input_rows;
input_rows.resize(block.nrows());
for (int i = 0; i < block.nrows(); i++) {
@@ -142,7 +142,8 @@ Status MajorDeltaCompaction::FlushRowSetAndDeltas() {
RETURN_NOT_OK(CopyRow(input_row.row, &dst_row, static_cast<Arena*>(nullptr)));
Mutation* new_undos_head = nullptr;
- // We're ignoring the result from new_redos_head because we'll find them later at step 5).
+ // We're ignoring the result from new_redos_head because we'll find them
+ // later at step 5.
Mutation* new_redos_head = nullptr;
bool is_garbage_collected;
@@ -176,8 +177,9 @@ Status MajorDeltaCompaction::FlushRowSetAndDeltas() {
// 4) Write the new base data.
RETURN_NOT_OK(base_data_writer_->AppendBlock(block));
- // 5) Remove the columns that we're compacting from the delta flush, but keep all the
- // delete mutations.
+ // 5) Remove the columns that we've done our major REDO delta compaction on
+ // from this delta flush, except keep all the delete and reinsert
+ // mutations.
arena.Reset();
vector<DeltaKeyAndUpdate> out;
RETURN_NOT_OK(delta_iter_->FilterColumnIdsAndCollectDeltas(column_ids_, &out, &arena));
@@ -187,7 +189,8 @@ Status MajorDeltaCompaction::FlushRowSetAndDeltas() {
RETURN_NOT_OK(OpenRedoDeltaFileWriter());
}
- // 6) Write the deltas we're not compacting back into a delta file.
+ // 6) Write the remaining REDO deltas that we haven't compacted away back
+ // into a REDO delta file.
for (const DeltaKeyAndUpdate& key_and_update : out) {
RowChangeList update(key_and_update.cell);
RETURN_NOT_OK_PREPEND(new_redo_delta_writer_->AppendDelta<REDO>(key_and_update.key, update),
@@ -260,7 +263,7 @@ Status MajorDeltaCompaction::Compact() {
LOG(INFO) << "Preparing to major compact delta file: " << ds->ToString();
}
- // We defer on calling OpenNewDeltaBlock since we might not need to flush.
+ // We defer calling OpenRedoDeltaFileWriter() since we might not need to flush.
RETURN_NOT_OK(OpenBaseDataWriter());
RETURN_NOT_OK(FlushRowSetAndDeltas());
LOG(INFO) << "Finished major delta compaction of columns " <<
[2/2] kudu git commit: tests: add --stress_cpu_threads argument
Posted by mp...@apache.org.
tests: add --stress_cpu_threads argument
This is equivalent to running 'stress -c <N>' except it's built into our
test binaries. This is quite handy for reproducing race conditions,
especially on dist-test where it isn't easy to just pop open another
terminal and run 'stress' at the same time.
Change-Id: I9214098a9f83fad78889b2ff9621b19109f92425
Reviewed-on: http://gerrit.cloudera.org:8080/4106
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/df3d070e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/df3d070e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/df3d070e
Branch: refs/heads/master
Commit: df3d070e569d33875dc5d64180acbf913a2e2288
Parents: 5350702
Author: Todd Lipcon <to...@apache.org>
Authored: Tue Aug 23 18:46:25 2016 -0700
Committer: Mike Percy <mp...@apache.org>
Committed: Wed Aug 24 18:47:18 2016 +0000
----------------------------------------------------------------------
src/kudu/util/test_main.cc | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/df3d070e/src/kudu/util/test_main.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/test_main.cc b/src/kudu/util/test_main.cc
index dd9af5f..f1b3763 100644
--- a/src/kudu/util/test_main.cc
+++ b/src/kudu/util/test_main.cc
@@ -20,7 +20,9 @@
#include <gtest/gtest.h>
#include <signal.h>
#include <sys/time.h>
+#include <thread>
+#include "kudu/gutil/atomicops.h"
#include "kudu/util/pstack_watcher.h"
#include "kudu/util/flags.h"
#include "kudu/util/status.h"
@@ -28,6 +30,10 @@
DEFINE_int32(test_timeout_after, 0,
"Maximum total seconds allowed for all unit tests in the suite. Default: disabled");
+DEFINE_int32(stress_cpu_threads, 0,
+ "Number of threads to start that burn CPU in an attempt to "
+ "stimulate race conditions");
+
// Start timer that kills the process if --test_timeout_after is exceeded before
// the tests complete.
static void CreateAndStartTimer();
@@ -35,6 +41,17 @@ static void CreateAndStartTimer();
// Gracefully kill the process.
static void KillTestOnTimeout(int signum);
+static void StartStressThreads() {
+ for (int i = 0; i < FLAGS_stress_cpu_threads; i++) {
+ std::thread([]{
+ while (true) {
+ // Do something which won't be optimized out.
+ base::subtle::MemoryBarrier();
+ }
+ }).detach();
+ }
+}
+
int main(int argc, char **argv) {
google::InstallFailureSignalHandler();
// InitGoogleTest() must precede ParseCommandLineFlags(), as the former
@@ -45,6 +62,8 @@ int main(int argc, char **argv) {
// Create the test-timeout timer.
CreateAndStartTimer();
+ StartStressThreads();
+
int ret = RUN_ALL_TESTS();
return ret;