You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by gr...@apache.org on 2018/10/18 21:20:12 UTC
[1/3] kudu git commit: tablet_server-test: wait for leader in some
tests
Repository: kudu
Updated Branches:
refs/heads/master 53e028798 -> 785490ce5
tablet_server-test: wait for leader in some tests
The fix for KUDU-2463 made successful Raft elections a prerequisite for
scanning empty tablets. Under enough stress, even in a single replica
case, this may not complete immediately, and can thus, lead scans to
return a different error than expected. This patch updates the test
fixture that creates an empty tablet in tablet_server-test to wait until
this has passed before deeming the tablet ready for business.
The following test incantation:
tablet_server-test --gtest_filter=*BadProjectionTypes* --stress_cpu_threads=100
would fail fairly consistently without this patch.
Change-Id: Ie6ca079353d4d44a48defc386d94b4bda452ec1b
Reviewed-on: http://gerrit.cloudera.org:8080/11718
Tested-by: Andrew Wong <aw...@cloudera.com>
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/64d5fe52
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/64d5fe52
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/64d5fe52
Branch: refs/heads/master
Commit: 64d5fe5267d3f936e9b366d6fa3de8e5a7a42212
Parents: 53e0287
Author: Andrew Wong <aw...@cloudera.com>
Authored: Wed Oct 17 16:20:38 2018 -0700
Committer: Andrew Wong <aw...@cloudera.com>
Committed: Thu Oct 18 18:47:50 2018 +0000
----------------------------------------------------------------------
src/kudu/tserver/tablet_server-test-base.cc | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/64d5fe52/src/kudu/tserver/tablet_server-test-base.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tserver/tablet_server-test-base.cc b/src/kudu/tserver/tablet_server-test-base.cc
index df68bbd..924bf5e 100644
--- a/src/kudu/tserver/tablet_server-test-base.cc
+++ b/src/kudu/tserver/tablet_server-test-base.cc
@@ -44,6 +44,7 @@
#include "kudu/rpc/messenger.h"
#include "kudu/rpc/rpc_controller.h"
#include "kudu/tablet/local_tablet_writer.h"
+#include "kudu/tablet/mvcc.h"
#include "kudu/tablet/tablet.h"
#include "kudu/tablet/tablet_replica.h"
#include "kudu/tserver/mini_tablet_server.h"
@@ -133,10 +134,22 @@ void TabletServerTestBase::StartTabletServer(int num_data_dirs) {
Status TabletServerTestBase::WaitForTabletRunning(const char *tablet_id) {
scoped_refptr<tablet::TabletReplica> tablet_replica;
const auto* tablet_manager = mini_server_->server()->tablet_manager();
+ const auto kTimeout = MonoDelta::FromSeconds(10);
RETURN_NOT_OK(tablet_manager->GetTabletReplica(tablet_id, &tablet_replica));
- RETURN_NOT_OK(tablet_replica->WaitUntilConsensusRunning(MonoDelta::FromSeconds(10)));
+ RETURN_NOT_OK(tablet_replica->WaitUntilConsensusRunning(kTimeout));
RETURN_NOT_OK(
- tablet_replica->consensus()->WaitUntilLeaderForTests(MonoDelta::FromSeconds(10)));
+ tablet_replica->consensus()->WaitUntilLeaderForTests(kTimeout));
+
+ // KUDU-2463: Even though the tablet thinks its leader, for correctness, it
+ // must wait to finish replicating its no-op (even as a single replica)
+ // before being available to scans.
+ MonoTime deadline = MonoTime::Now() + kTimeout;
+ while (!tablet_replica->tablet()->mvcc_manager()->CheckIsSafeTimeInitialized().ok()) {
+ if (MonoTime::Now() >= deadline) {
+ return Status::TimedOut("mvcc did not advance safe time within timeout");
+ }
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ }
// KUDU-2444: Even though the tablet replica is fully running, the tablet
// manager may regard it as still transitioning to the running state.
[2/3] kudu git commit: [tools] updated comments on 'kudu perf loadgen'
Posted by gr...@apache.org.
[tools] updated comments on 'kudu perf loadgen'
While using the 'kudu perf loadgen' tool recently, I noticed that the
descriptions for a few of its flags were not clear enough. This changelist
fixes that and also updates the examples of usage since some of those had
the masters' RPC end-point argument misplaced.
This changelist does not contain any functional changes.
Change-Id: If171957db9d9be8d14b5031a64f4edbfa5470be0
Reviewed-on: http://gerrit.cloudera.org:8080/11724
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1d34ae66
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1d34ae66
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1d34ae66
Branch: refs/heads/master
Commit: 1d34ae66524cc5a61bd7e0edf942039d989a5c09
Parents: 64d5fe5
Author: Alexey Serbin <as...@cloudera.com>
Authored: Thu Oct 18 01:17:11 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Thu Oct 18 21:06:59 2018 +0000
----------------------------------------------------------------------
src/kudu/tools/tool_action_perf.cc | 70 +++++++++++++++++----------------
1 file changed, 37 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/1d34ae66/src/kudu/tools/tool_action_perf.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_perf.cc b/src/kudu/tools/tool_action_perf.cc
index d162bbf..e86e5c0 100644
--- a/src/kudu/tools/tool_action_perf.cc
+++ b/src/kudu/tools/tool_action_perf.cc
@@ -16,11 +16,15 @@
// under the License.
//
-// This is a small load generation tool which pushes data to a tablet
-// server as fast as possible. The table is supposed to be created already,
-// and this tool populates it with generated data. As an option, it's possible
-// to run a post-scan over the inserted rows to get total table row count
-// as reported by the scan operation.
+// This is a small load generation tool which generates and pushes data
+// to a Kudu cluster as fast as possible. In each run, the tool inserts data
+// into a single table. The table might exist already, and the tool then
+// populates it with generated data in accordance with the table schema.
+// The tool can also create a dedicated temporary table with pre-defined
+// schema. By default, dedicated temporary tables are dropped upon successful
+// completion of the tool, but they can be optionally preserved. Also,
+// a post-insertion scan may be run on the table to verify that the count
+// of the inserted rows matches the expected number.
//
// See below for examples of usage.
//
@@ -30,14 +34,13 @@
// of length 64 for binary and string fields
// with Kudu master server listening on the default port at localhost:
//
-// kudu perf loadgen \
+// kudu perf loadgen 127.0.0.1 \
// --num_threads=1 \
// --num_rows_per_thread=8000000 \
// --string_len=64 \
// --buffer_size_bytes=33554432 \
// --buffers_num=0 \
// --flush_per_n_rows=2000 \
-// 127.0.0.1
//
//
// Run in AUTO_FLUSH_BACKGROUND mode, 2 threads inserting 4M rows each inserting
@@ -46,31 +49,29 @@
// using the specified pre-set string for binary and string fields
// with Kudu master server listening on the default port at localhost:
//
-// kudu perf loadgen \
+// kudu perf loadgen 127.0.0.1 \
// --num_threads=2 \
// --num_rows_per_thread=4000000 \
// --string_fixed=012345678901234567890123456789012 \
// --buffer_size_bytes=1048576 \
// --buffer_flush_watermark_pct=0.125 \
-// --buffers_num=8 \
-// 127.0.0.1
+// --buffers_num=8
//
//
// Run in AUTO_FLUSH_BACKGROUND mode, 4 threads inserting 2M rows each inserting
// into auto-created table, with limit of 4 buffers max 64KB in size total,
// having 25% for buffer flush watermark,
// using the specified pre-set string for binary and string fields
-// with Kudu master server listening at 127.0.0.1:8765
+// with Kudu master server listening at default master RPC port at 127.0.0.1:
//
-// kudu perf loadgen \
+// kudu perf loadgen 127.0.0.1 \
// --num_threads=4 \
// --num_rows_per_thread=2000000 \
// --string_fixed=0123456789 \
// --buffer_size_bytes=65536 \
// --buffers_num=4 \
// --buffer_flush_watermark_pct=0.25 \
-// --table_name=bench_02 \
-// 127.0.0.1:8765
+// --table_name=bench_02
//
//
// Run with default parameter values for data generation and batching,
@@ -79,9 +80,8 @@
// plus run post-insertion row scan to verify
// that the count of the inserted rows matches the expected number:
//
-// kudu perf loadgen \
-// --run_scan=true \
-// 127.0.0.1
+// kudu perf loadgen 127.0.0.1 \
+// --run_scan=true
//
//
// If running the tool against already existing table multiple times,
@@ -108,7 +108,8 @@
// partitioning, or both. Below are a few examples of this.
//
// kudu perf loadgen 127.0.0.1 --num_threads=8 --num_rows_per_thread=1000 \
-// --table_num_hash_partitions=1 --table_num_range_partitions=8 --use_random=false
+// --table_num_hash_partitions=1 --table_num_range_partitions=8 \
+// --use_random=false
//
// In the above example, a table with eight range partitions will be created,
// each partition will be in charge of 1000 rows worth of values; for a
@@ -118,12 +119,14 @@
// into a single range partition.
//
// kudu perf loadgen 127.0.0.1 --num_threads=8 --num_rows_per_thread=1000 \
-// --table_num_hash_partitions=8 --table_num_range_partitions=1 --use_random=false
+// --table_num_hash_partitions=8 --table_num_range_partitions=1 \
+// --use_random=false
//
// In the above example, a table with 8 hash partitions will be created.
//
// kudu perf loadgen 127.0.0.1 --num_threads=8 --num_rows_per_thread=1000 \
-// --table_num_hash_partitions=8 --table_num_range_partitions=8 --use_random=false
+// --table_num_hash_partitions=8 --table_num_range_partitions=8 \
+// --use_random=false
//
// In the above example, a table with a total of 64 tablets will be created.
// The range partitioning splits will be the same as those in the
@@ -255,8 +258,8 @@ DEFINE_bool(keep_auto_table, false,
"finishes. By default, the auto-generated table is dropped "
"after successfully finishing the test. NOTE: this parameter "
"has no effect if using already existing table "
- "(see the '--table_name' flag): the existing tables nor their data "
- "are never dropped/deleted.");
+ "(see the '--table_name' flag): neither the existing table "
+ "nor its data is ever dropped/deleted.");
DEFINE_uint64(num_rows_per_thread, 1000,
"Number of rows each thread generates and inserts; "
"0 means unlimited. All rows generated by a thread are inserted "
@@ -771,17 +774,18 @@ unique_ptr<Mode> BuildPerfMode() {
.AddOptionalParameter("string_fixed")
.AddOptionalParameter("string_len")
.AddOptionalParameter("table_name", boost::none, string(
- "Name of an existing table to use for the test. The test will "
- "determine the structure of the table schema and "
- "populate it with data accordingly. If left empty, "
- "the test automatically creates a table of pre-defined columnar "
- "structure with unique name and uses it to insert "
- "auto-generated data. The auto-created table is dropped "
- "upon successful completion of the test if not overridden "
- "by the '--keep_auto_table' flag. If running the test against "
- "an already existing table, it's highly recommended to use a "
- "dedicated table created just for testing purposes: "
- "the existing table nor its data is never dropped/deleted."))
+ "Name of an existing table to use for the test. The test will "
+ "determine the structure of the table schema and "
+ "populate it with data accordingly. If left empty, "
+ "the test automatically creates a table of pre-defined columnar "
+ "structure with unique name and uses it to insert "
+ "auto-generated data. The auto-created table is dropped "
+ "upon successful completion of the test if not overridden "
+ "by the '--keep_auto_table' flag. If running the test against "
+ "an already existing table, it's recommended to use a dedicated "
+ "table created just for testing purposes: the tool doesn't delete "
+ "the rows it inserted into the table. Neither the existing table "
+ "nor its data is ever dropped/deleted."))
.AddOptionalParameter("table_num_hash_partitions")
.AddOptionalParameter("table_num_range_partitions")
.AddOptionalParameter("table_num_replicas")
[3/3] kudu git commit: [java] Allow command line override of
kuduBinDir
Posted by gr...@apache.org.
[java] Allow command line override of kuduBinDir
This patch allows overriding the kuduBinDir in the
Gradle build by passing -DkuduBinDir or -PkuduBinDir
when running `gradle test`.
Change-Id: Icd7e60da01bae8aa6677e17dc4193ace26078a11
Reviewed-on: http://gerrit.cloudera.org:8080/11725
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/785490ce
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/785490ce
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/785490ce
Branch: refs/heads/master
Commit: 785490ce509e68029f8062882bb7021895e3b446
Parents: 1d34ae6
Author: Grant Henke <gr...@apache.org>
Authored: Thu Oct 18 14:05:18 2018 -0500
Committer: Grant Henke <gr...@apache.org>
Committed: Thu Oct 18 21:11:31 2018 +0000
----------------------------------------------------------------------
java/gradle/tests.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/785490ce/java/gradle/tests.gradle
----------------------------------------------------------------------
diff --git a/java/gradle/tests.gradle b/java/gradle/tests.gradle
index d356ad0..0da59ec 100644
--- a/java/gradle/tests.gradle
+++ b/java/gradle/tests.gradle
@@ -61,7 +61,7 @@ tasks.withType(Test) {
systemProperty "rerunFailingTestsCount", propertyWithDefault("rerunFailingTestsCount", 0)
// Set kuduBinDir to the binaries to use with the MiniKuduCluster.
- systemProperty "kuduBinDir", "$project.rootDir/../build/latest/bin"
+ systemProperty "kuduBinDir", propertyWithDefault("kuduBinDir", "$project.rootDir/../build/latest/bin")
// Don't fail the build if a --tests filter doesn't match any tests.
// This is useful for filtering tests from the top of the project where some