You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by to...@apache.org on 2016/02/02 07:55:24 UTC
[1/9] incubator-kudu git commit: KUDU-1286. Set default replication
to 3
Repository: incubator-kudu
Updated Branches:
refs/heads/master 30773866c -> 0b5545bc3
KUDU-1286. Set default replication to 3
Simple change but required fixing a bunch of tests.
Change-Id: I3de9f66ee25cf60be26b4f822894e31915805ecc
Reviewed-on: http://gerrit.cloudera.org:8080/1959
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/7e34050c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/7e34050c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/7e34050c
Branch: refs/heads/master
Commit: 7e34050c1b35cd51f39fe228aabc7ed4a9a2db7a
Parents: 3077386
Author: Jean-Daniel Cryans <jd...@cloudera.com>
Authored: Fri Jan 29 10:23:13 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Sat Jan 30 02:18:44 2016 +0000
----------------------------------------------------------------------
.../src/main/java/org/kududb/client/CreateTableOptions.java | 5 +++--
.../src/test/java/org/kududb/client/TestAsyncKuduClient.java | 6 ++++--
.../src/test/java/org/kududb/client/TestKuduTable.java | 2 +-
src/kudu/benchmarks/tpch/rpc_line_item_dao.cc | 1 +
src/kudu/client/client-test.cc | 8 ++------
src/kudu/client/client_samples-test.sh | 1 +
src/kudu/integration-tests/full_stack-insert-scan-test.cc | 1 +
src/kudu/integration-tests/update_scan_delta_compact-test.cc | 1 +
src/kudu/master/catalog_manager.cc | 4 ++--
src/kudu/master/master-test-util.h | 1 +
10 files changed, 17 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java b/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
index b9b509d..da8c2bc 100644
--- a/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
+++ b/java/kudu-client/src/main/java/org/kududb/client/CreateTableOptions.java
@@ -112,8 +112,9 @@ public class CreateTableOptions {
}
/**
- * Sets the number of replicas that each tablet will have. If not specified, it defaults to 1
- * replica which isn't safe for production usage.
+ * Sets the number of replicas that each tablet will have. If not specified, it uses the
+ * server-side default which is usually 3 unless changed by an administrator.
+ *
* @param numReplicas the number of replicas to use
* @return this instance
*/
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
index 55bbee0..7b4c932 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestAsyncKuduClient.java
@@ -36,7 +36,9 @@ public class TestAsyncKuduClient extends BaseKuduTest {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
BaseKuduTest.setUpBeforeClass();
- table = createTable(TABLE_NAME, basicSchema, new CreateTableOptions());
+ // Set to 1 for testDisconnect to always test disconnecting the right server.
+ CreateTableOptions options = new CreateTableOptions().setNumReplicas(1);
+ table = createTable(TABLE_NAME, basicSchema, options);
}
@Test(timeout = 100000)
@@ -130,4 +132,4 @@ public class TestAsyncKuduClient extends BaseKuduTest {
assertTrue(ex.getMessage().contains(badHostname));
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
index b19753b..b158aa7 100644
--- a/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/kududb/client/TestKuduTable.java
@@ -241,7 +241,7 @@ public class TestKuduTable extends BaseKuduTest {
assertEquals(splitsCount + 1, tablets.size());
assertEquals(splitsCount + 1, table.asyncGetTabletsLocations(DEFAULT_SLEEP).join().size());
for (LocatedTablet tablet : tablets) {
- assertEquals(1, tablet.getReplicas().size());
+ assertEquals(3, tablet.getReplicas().size());
}
return table;
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/benchmarks/tpch/rpc_line_item_dao.cc
----------------------------------------------------------------------
diff --git a/src/kudu/benchmarks/tpch/rpc_line_item_dao.cc b/src/kudu/benchmarks/tpch/rpc_line_item_dao.cc
index e90801a..12d1c3f 100644
--- a/src/kudu/benchmarks/tpch/rpc_line_item_dao.cc
+++ b/src/kudu/benchmarks/tpch/rpc_line_item_dao.cc
@@ -110,6 +110,7 @@ void RpcLineItemDAO::Init() {
gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
CHECK_OK(table_creator->table_name(table_name_)
.schema(&schema)
+ .num_replicas(1)
.split_rows(tablet_splits_)
.Create());
CHECK_OK(client_->OpenTable(table_name_, &client_table_));
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/client/client-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index 172a634..032000b 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -598,13 +598,8 @@ TEST_F(ClientTest, TestScanMultiTablet) {
rows.push_back(row);
}
gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
- ASSERT_OK(table_creator->table_name("TestScanMultiTablet")
- .schema(&schema_)
- .split_rows(rows)
- .Create());
-
shared_ptr<KuduTable> table;
- ASSERT_OK(client_->OpenTable("TestScanMultiTablet", &table));
+ ASSERT_NO_FATAL_FAILURE(CreateTable("TestScanMultiTablet", 1, rows, &table));
// Insert rows with keys 12, 13, 15, 17, 22, 23, 25, 27...47 into each
// tablet, except the first which is empty.
@@ -2544,6 +2539,7 @@ TEST_F(ClientTest, TestCreateDuplicateTable) {
gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
ASSERT_TRUE(table_creator->table_name(kTableName)
.schema(&schema_)
+ .num_replicas(1)
.Create().IsAlreadyPresent());
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/client/client_samples-test.sh
----------------------------------------------------------------------
diff --git a/src/kudu/client/client_samples-test.sh b/src/kudu/client/client_samples-test.sh
index 4624196..5f0d9fc 100755
--- a/src/kudu/client/client_samples-test.sh
+++ b/src/kudu/client/client_samples-test.sh
@@ -102,6 +102,7 @@ export TEST_TMPDIR=${TEST_TMPDIR:-$TMPDIR/kudutest-$UID}
mkdir -p $TEST_TMPDIR
BASE_DIR=$(mktemp -d $TEST_TMPDIR/client_samples-test.XXXXXXXX)
$OUTPUT_DIR/kudu-master \
+ --default_num_replicas=1 \
--log_dir=$BASE_DIR \
--fs_wal_dir=$BASE_DIR/master \
--fs_data_dirs=$BASE_DIR/master \
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/integration-tests/full_stack-insert-scan-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/full_stack-insert-scan-test.cc b/src/kudu/integration-tests/full_stack-insert-scan-test.cc
index a2abf77..3a07af1 100644
--- a/src/kudu/integration-tests/full_stack-insert-scan-test.cc
+++ b/src/kudu/integration-tests/full_stack-insert-scan-test.cc
@@ -134,6 +134,7 @@ class FullStackInsertScanTest : public KuduTest {
gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
ASSERT_OK(table_creator->table_name(kTableName)
.schema(&schema_)
+ .num_replicas(1)
.Create());
ASSERT_OK(client_->OpenTable(kTableName, &reader_table_));
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/integration-tests/update_scan_delta_compact-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/update_scan_delta_compact-test.cc b/src/kudu/integration-tests/update_scan_delta_compact-test.cc
index 82b3edd..a9aa81a 100644
--- a/src/kudu/integration-tests/update_scan_delta_compact-test.cc
+++ b/src/kudu/integration-tests/update_scan_delta_compact-test.cc
@@ -85,6 +85,7 @@ class UpdateScanDeltaCompactionTest : public KuduTest {
gscoped_ptr<KuduTableCreator> table_creator(client_->NewTableCreator());
ASSERT_OK(table_creator->table_name(kTableName)
.schema(&schema_)
+ .num_replicas(1)
.Create());
ASSERT_OK(client_->OpenTable(kTableName, &table_));
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 3d7867d..e1ecadb 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -105,8 +105,8 @@ DEFINE_int32(unresponsive_ts_rpc_timeout_ms, 60 * 60 * 1000, // 1 hour
"a tablet server in order to perform operations such as deleting a tablet.");
TAG_FLAG(unresponsive_ts_rpc_timeout_ms, advanced);
-DEFINE_int32(default_num_replicas, 1, // TODO switch to 3 and fix SelectReplicas()
- "Default number of replicas for tables that do have the num_replicas set.");
+DEFINE_int32(default_num_replicas, 3,
+ "Default number of replicas for tables that do not have the num_replicas set.");
TAG_FLAG(default_num_replicas, advanced);
DEFINE_int32(catalog_manager_bg_task_wait_ms, 1000,
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7e34050c/src/kudu/master/master-test-util.h
----------------------------------------------------------------------
diff --git a/src/kudu/master/master-test-util.h b/src/kudu/master/master-test-util.h
index d74bbb9..1ee264c 100644
--- a/src/kudu/master/master-test-util.h
+++ b/src/kudu/master/master-test-util.h
@@ -79,6 +79,7 @@ void CreateTabletForTesting(MiniMaster* mini_master,
CreateTableResponsePB resp;
req.set_name(table_name);
+ req.set_num_replicas(1);
ASSERT_OK(SchemaToPB(schema, req.mutable_schema()));
ASSERT_OK(mini_master->master()->catalog_manager()->CreateTable(&req, &resp, NULL));
}
[8/9] incubator-kudu git commit: env_posix: remove unused page_size_
member
Posted by to...@apache.org.
env_posix: remove unused page_size_ member
Change-Id: I4d0d9e6470fc618ec60f325c25852f086f919336
Reviewed-on: http://gerrit.cloudera.org:8080/1982
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/e078f77e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/e078f77e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/e078f77e
Branch: refs/heads/master
Commit: e078f77e159a50d5c11788da59262e0f27831b47
Parents: 197b5fa
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Feb 1 15:43:50 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Feb 2 02:55:46 2016 +0000
----------------------------------------------------------------------
src/kudu/util/env_posix.cc | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/e078f77e/src/kudu/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index f55e83c..d568f76 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -1121,12 +1121,9 @@ class PosixEnv : public Env {
return Status::OK();
}
}
-
- size_t page_size_;
};
-PosixEnv::PosixEnv() : page_size_(getpagesize()) {
-}
+PosixEnv::PosixEnv() {}
} // namespace
[4/9] incubator-kudu git commit: Revert "Remove autoreconf calls in
download-thirdparty.sh"
Posted by to...@apache.org.
Revert "Remove autoreconf calls in download-thirdparty.sh"
This reverts commit ee13c00a5df5d4ad1cac05b49048a9eae256e2f2.
Change-Id: Ifb5f09e9b4e710585cba6d7b34c198b549884eb8
Reviewed-on: http://gerrit.cloudera.org:8080/1972
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/4cc1a03b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/4cc1a03b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/4cc1a03b
Branch: refs/heads/master
Commit: 4cc1a03b9c9ca2dd0452960ded6e0c541e58df14
Parents: 7a68e1f
Author: Dan Burkert <da...@cloudera.com>
Authored: Sat Jan 30 03:12:18 2016 +0000
Committer: Dan Burkert <da...@cloudera.com>
Committed: Sat Jan 30 17:19:04 2016 +0000
----------------------------------------------------------------------
thirdparty/download-thirdparty.sh | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/4cc1a03b/thirdparty/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index f50eb9f..575ea19 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -17,6 +17,9 @@
# specific language governing permissions and limitations
# under the License.
+# autoreconf calls are necessary to fix hard-coded aclocal versions in the
+# configure scripts that ship with the projects.
+
set -e
TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
@@ -71,6 +74,7 @@ if [ ! -d $GLOG_DIR ]; then
pushd $GLOG_DIR
patch -p0 < $TP_DIR/patches/glog-issue-198-fix-unused-warnings.patch
touch patchlevel-$GLOG_PATCHLEVEL
+ autoreconf -fvi
popd
echo
fi
@@ -95,12 +99,16 @@ if [ ! -d $GPERFTOOLS_DIR ]; then
patch -p1 < $TP_DIR/patches/gperftools-Change-default-TCMALLOC_TRANSFER_NUM_OBJ-to-40.patch
patch -p1 < $TP_DIR/patches/gperftools-hook-mi_force_unlock-on-OSX-instead-of-pthread_atfork.patch
touch patchlevel-$GPERFTOOLS_PATCHLEVEL
+ autoreconf -fvi
popd
echo
fi
if [ ! -d $PROTOBUF_DIR ]; then
fetch_and_expand protobuf-${PROTOBUF_VERSION}.tar.gz
+ pushd $PROTOBUF_DIR
+ autoreconf -fvi
+ popd
fi
if [ ! -d $CMAKE_DIR ]; then
@@ -109,6 +117,9 @@ fi
if [ ! -d $SNAPPY_DIR ]; then
fetch_and_expand snappy-${SNAPPY_VERSION}.tar.gz
+ pushd $SNAPPY_DIR
+ autoreconf -fvi
+ popd
fi
if [ ! -d $ZLIB_DIR ]; then
[6/9] incubator-kudu git commit: Do not report tests that fail due to
NTP issues
Posted by to...@apache.org.
Do not report tests that fail due to NTP issues
Occasionally we have tests that fail due to machines with broken
NTP synchronization. In those cases, we don't want to report them
to the flaky test infrastructure, since it inflates our flakiness
count on every test which uses NTP.
Change-Id: Ia103868de0e6fd3ef95ced72035de7f67ccfb8db
Reviewed-on: http://gerrit.cloudera.org:8080/1980
Reviewed-by: Jean-Daniel Cryans
Tested-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/64bd51c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/64bd51c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/64bd51c5
Branch: refs/heads/master
Commit: 64bd51c5795ac9726734f29b4060e9fae9dd2b72
Parents: cecdb47
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Feb 1 15:28:32 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Feb 2 02:00:19 2016 +0000
----------------------------------------------------------------------
build-support/report-test.sh | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/64bd51c5/build-support/report-test.sh
----------------------------------------------------------------------
diff --git a/build-support/report-test.sh b/build-support/report-test.sh
index 34872c3..b0d9426 100755
--- a/build-support/report-test.sh
+++ b/build-support/report-test.sh
@@ -73,6 +73,13 @@ if [ -n "$HEAPCHECK" ]; then
BUILD_CONFIG="$BUILD_CONFIG heapcheck"
fi
+# We sometimes have flaky infrastructure where NTP is broken. In that case
+# do not report it as a failed test.
+if grep -q 'Clock considered unsynchronized' $LOGFILE ; then
+ echo Not reporting test that failed due to NTP issues.
+ exit 1
+fi
+
# Only upload a log if the test failed.
# This saves some space on S3, network bandwidth, etc, and we don't
# have a lot of use for the logs of successful tests anyway.
[2/9] incubator-kudu git commit: Remove autoreconf calls in
download-thirdparty.sh
Posted by to...@apache.org.
Remove autoreconf calls in download-thirdparty.sh
The original reason we added these (changing the stdlib on OS X) is no longer a
concern, so it can be removed.
Change-Id: I85c9053729facb0a50b290844a84099e7cff46f5
Reviewed-on: http://gerrit.cloudera.org:8080/1969
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/ee13c00a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/ee13c00a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/ee13c00a
Branch: refs/heads/master
Commit: ee13c00a5df5d4ad1cac05b49048a9eae256e2f2
Parents: 7e34050
Author: Dan Burkert <da...@cloudera.com>
Authored: Fri Jan 29 17:40:02 2016 -0800
Committer: Dan Burkert <da...@cloudera.com>
Committed: Sat Jan 30 02:53:05 2016 +0000
----------------------------------------------------------------------
thirdparty/download-thirdparty.sh | 34 ----------------------------------
1 file changed, 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/ee13c00a/thirdparty/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 2cd7889..2bdd519 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -16,29 +16,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-# Note regarding the autoreconf calls:
-#
-# Mac OS 10.8 shipped a clang-based C++ toolchain that provides both GNU libstdc++ and LLVM's
-# libc++. The default library used is libstdc++, though one can override that with the
-# -stdlib=libc++ option to clang (while compiling and linking). In 10.9, the default policy has
-# switched: libc++ is now the default, and to use libstdc++ you need to pass -stdlib=libstdc++
-# as a command line option.
-#
-# This is relevant to Kudu because libc++ does not support tr1; to use tr1 features like tr1/memory
-# we must use them from the C++11 namespace (i.e. <memory> instead of <tr1/memory> and -std=c++11
-# to clang).
-#
-# Setting CXXFLAGS=-stdlib=libstdc++ suffices for an autotools-based project, and this is what we do
-# in build-thirdparty.sh. However, older versions of autotools will filter out -stdlib=libstdc++
-# from a shared library link invocation. This leads to link failures with every std symbol listed
-# as "undefined". To fix this, one must regenerate the autotools system for each library on a
-# machine with modern brews of autotools. Running "autoconf -fvi" inside the library's directory
-# is sufficient. See this link for more information:
-#
-# http://trac.macports.org/ticket/32982
-#
-# This is why all the cmake-based projects have their autotools regenerated with "autoreconf -fvi".
set -e
@@ -94,7 +71,6 @@ if [ ! -d $GLOG_DIR ]; then
pushd $GLOG_DIR
patch -p0 < $TP_DIR/patches/glog-issue-198-fix-unused-warnings.patch
touch patchlevel-$GLOG_PATCHLEVEL
- autoreconf -fvi
popd
echo
fi
@@ -105,9 +81,6 @@ fi
if [ ! -d $GFLAGS_DIR ]; then
fetch_and_expand gflags-${GFLAGS_VERSION}.zip
- pushd $GFLAGS_DIR
- autoreconf -fvi
- popd
fi
# Check that the gperftools patch has been applied.
@@ -122,16 +95,12 @@ if [ ! -d $GPERFTOOLS_DIR ]; then
patch -p1 < $TP_DIR/patches/gperftools-Change-default-TCMALLOC_TRANSFER_NUM_OBJ-to-40.patch
patch -p1 < $TP_DIR/patches/gperftools-hook-mi_force_unlock-on-OSX-instead-of-pthread_atfork.patch
touch patchlevel-$GPERFTOOLS_PATCHLEVEL
- autoreconf -fvi
popd
echo
fi
if [ ! -d $PROTOBUF_DIR ]; then
fetch_and_expand protobuf-${PROTOBUF_VERSION}.tar.gz
- pushd $PROTOBUF_DIR
- autoreconf -fvi
- popd
fi
if [ ! -d $CMAKE_DIR ]; then
@@ -140,9 +109,6 @@ fi
if [ ! -d $SNAPPY_DIR ]; then
fetch_and_expand snappy-${SNAPPY_VERSION}.tar.gz
- pushd $SNAPPY_DIR
- autoreconf -fvi
- popd
fi
if [ ! -d $ZLIB_DIR ]; then
[7/9] incubator-kudu git commit: Enable compression and smaller block
size for composite key index
Posted by to...@apache.org.
Enable compression and smaller block size for composite key index
After running the time series workload on d2106 for a couple months,
I found a couple interesting things:
- The composite key index (aka "ad hoc index") was taking 6.4 bytes
per row (vs 5.23 *bits* for the actual data). Compressing it with
'lzop' on that dataset gained a 6.2x savings.
Thus, this patch changes this index to be compressed using LZ4 by default,
which should save space. On the tpch lineitem table, it saved about 15%.
The performance cost should be fairly minimal -- we always random-access
the index blocks, and in the case of a cache miss, the cost of decompression
is tiny compared to the cost of the resulting disk seek.
- Once we reached ~12B rows, the system degenerated into a seeky mess.
Looking at tracing revealed that we spent a lot of time reading
composite indexes, indicating they weren't fitting well in the cache.
I theorize that making these index blocks smaller should decrease the
amount of excess data that gets pulled into the cache when we read them.
Given that these blocks are always random-accessed and never scanned,
using small block sizes makes intuitive sense.
Eventually, both of these options should be table properties, but it
was easier to just set better defaults for now as a quick improvement.
Change-Id: I2b7bfc7a4961c764d262524292ec56e3969af728
Reviewed-on: http://gerrit.cloudera.org:8080/953
Reviewed-by: Jean-Daniel Cryans
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/197b5fa1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/197b5fa1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/197b5fa1
Branch: refs/heads/master
Commit: 197b5fa1a754510a01ac934e8ed5b966de8da29f
Parents: 64bd51c
Author: Todd Lipcon <to...@cloudera.com>
Authored: Tue Aug 11 18:32:07 2015 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Feb 2 02:55:13 2016 +0000
----------------------------------------------------------------------
src/kudu/tablet/diskrowset.cc | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/197b5fa1/src/kudu/tablet/diskrowset.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tablet/diskrowset.cc b/src/kudu/tablet/diskrowset.cc
index d397e82..55decd8 100644
--- a/src/kudu/tablet/diskrowset.cc
+++ b/src/kudu/tablet/diskrowset.cc
@@ -50,6 +50,10 @@ DEFINE_double(tablet_delta_store_major_compact_min_ratio, 0.1f,
"can run (Advanced option)");
TAG_FLAG(tablet_delta_store_major_compact_min_ratio, experimental);
+DEFINE_int32(default_composite_key_index_block_size_bytes, 4096,
+ "Block size used for composite key indexes.");
+TAG_FLAG(default_composite_key_index_block_size_bytes, experimental);
+
namespace kudu {
namespace tablet {
@@ -128,6 +132,8 @@ Status DiskRowSetWriter::InitAdHocIndexWriter() {
opts.write_posidx = false;
opts.storage_attributes.encoding = PREFIX_ENCODING;
+ opts.storage_attributes.compression = LZ4;
+ opts.storage_attributes.cfile_block_size = FLAGS_default_composite_key_index_block_size_bytes;
// Create the CFile writer for the ad-hoc index.
ad_hoc_index_writer_.reset(new cfile::CFileWriter(
[3/9] incubator-kudu git commit: Bump gflags version
Posted by to...@apache.org.
Bump gflags version
The previous version ran into into an issue building on OS X 10.11 due to a
mismatch in libtool versions. Bumping gflags to the latest version allows us to
sidestep this by using cmake.
Change-Id: Ifed00bf0396ceb6001f043cd3cd6b12c449cdfd2
Reviewed-on: http://gerrit.cloudera.org:8080/1970
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/7a68e1f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/7a68e1f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/7a68e1f8
Branch: refs/heads/master
Commit: 7a68e1f8baee04eaf259f7567ddc39e09a0f0938
Parents: ee13c00
Author: Dan Burkert <da...@cloudera.com>
Authored: Fri Jan 29 17:44:18 2016 -0800
Committer: Dan Burkert <da...@cloudera.com>
Committed: Sat Jan 30 02:54:36 2016 +0000
----------------------------------------------------------------------
src/kudu/tools/kudu-ksck.cc | 2 +-
thirdparty/build-definitions.sh | 13 ++++++++-----
thirdparty/download-thirdparty.sh | 2 +-
thirdparty/vars.sh | 2 +-
4 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7a68e1f8/src/kudu/tools/kudu-ksck.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-ksck.cc b/src/kudu/tools/kudu-ksck.cc
index 86f4a3a..a663750 100644
--- a/src/kudu/tools/kudu-ksck.cc
+++ b/src/kudu/tools/kudu-ksck.cc
@@ -19,8 +19,8 @@
// on the default RPC port. It verifies that all the reported Tablet Servers are running and that
// the tablets are in a consistent state.
+#include <gflags/gflags.h>
#include <glog/logging.h>
-#include <google/gflags.h>
#include <iostream>
#include "kudu/gutil/strings/split.h"
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7a68e1f8/thirdparty/build-definitions.sh
----------------------------------------------------------------------
diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh
index ed570f7..dc4d47e 100644
--- a/thirdparty/build-definitions.sh
+++ b/thirdparty/build-definitions.sh
@@ -115,11 +115,14 @@ build_libstdcxx() {
build_gflags() {
cd $GFLAGS_DIR
- CFLAGS="$EXTRA_CFLAGS" \
- CXXFLAGS="$EXTRA_CXXFLAGS" \
- LDFLAGS="$EXTRA_LDFLAGS" \
- LIBS="$EXTRA_LIBS" \
- ./configure --with-pic --prefix=$PREFIX
+ rm -rf CMakeCache.txt CMakeFiles/
+ CXXFLAGS="$EXTRA_CFLAGS $EXTRA_CXXFLAGS $EXTRA_LDFLAGS $EXTRA_LIBS" \
+ cmake \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=On \
+ -DCMAKE_INSTALL_PREFIX=$PREFIX \
+ -DBUILD_SHARED_LIBS=On \
+ -DBUILD_STATIC_LIBS=On
make -j$PARALLEL install
}
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7a68e1f8/thirdparty/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 2bdd519..f50eb9f 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -80,7 +80,7 @@ if [ ! -d $GMOCK_DIR ]; then
fi
if [ ! -d $GFLAGS_DIR ]; then
- fetch_and_expand gflags-${GFLAGS_VERSION}.zip
+ fetch_and_expand gflags-${GFLAGS_VERSION}.tar.gz
fi
# Check that the gperftools patch has been applied.
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/7a68e1f8/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index baeeec1..9dfcac1 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -29,7 +29,7 @@ PREFIX_DEPS_TSAN=$TP_DIR/installed-deps-tsan
PREFIX_LIBSTDCXX=$PREFIX_DEPS/gcc
PREFIX_LIBSTDCXX_TSAN=$PREFIX_DEPS_TSAN/gcc
-GFLAGS_VERSION=1.5
+GFLAGS_VERSION=2.1.2
GFLAGS_DIR=$TP_DIR/gflags-$GFLAGS_VERSION
GLOG_VERSION=0.3.4
[5/9] incubator-kudu git commit: python: start master with
default_num_replicas=1
Posted by to...@apache.org.
python: start master with default_num_replicas=1
The python tests became flaky after we changed the default number of replicas
to 3, because they assume read-your-writes semantics. With multiple replicas,
we don't guarantee this quite yet unless you specifically ask for it.
Change-Id: Ied02e8c1e0f3d06ff03509d846548e94667399ee
Reviewed-on: http://gerrit.cloudera.org:8080/1984
Reviewed-by: Jean-Daniel Cryans
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/cecdb47c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/cecdb47c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/cecdb47c
Branch: refs/heads/master
Commit: cecdb47ce4887bf4903acd825e26158b200e14e9
Parents: 4cc1a03
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Feb 1 15:57:50 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Feb 2 01:58:49 2016 +0000
----------------------------------------------------------------------
python/kudu/tests/common.py | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/cecdb47c/python/kudu/tests/common.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py
index 988b7d7..ba01260 100644
--- a/python/kudu/tests/common.py
+++ b/python/kudu/tests/common.py
@@ -60,6 +60,9 @@ class KuduTestBase(object):
"-log_dir={0}/master/logs".format(local_path),
"-logtostderr",
"-webserver_port=0",
+ # Only make one replica so that our tests don't need to worry about
+ # setting consistency modes.
+ "-default_num_replicas=1",
"-server_dump_info_path={0}/master/config.json".format(local_path)
]
[9/9] incubator-kudu git commit: KUDU-1317 (part 2): decay the
replica creation load on TS lazily
Posted by to...@apache.org.
KUDU-1317 (part 2): decay the replica creation load on TS lazily
This is a follow-up in response to a review by Binglin[1] suggesting
that the load value for TS Descriptors should be decayed on access
instead of in a background loop. This also adds a new test verifying
the behavior.
[1] http://gerrit.cloudera.org:8080/1654
Change-Id: I15899b8bf221f334fdec5983e1b5d93cc57fab3b
Reviewed-on: http://gerrit.cloudera.org:8080/1979
Reviewed-by: Jean-Daniel Cryans
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Todd Lipcon <to...@apache.org>
Reviewed-by: Binglin Chang <de...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/0b5545bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/0b5545bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/0b5545bc
Branch: refs/heads/master
Commit: 0b5545bc30518f895dd4c6936188831453252a7f
Parents: e078f77
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Feb 1 14:55:33 2016 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Feb 2 06:54:55 2016 +0000
----------------------------------------------------------------------
src/kudu/master/catalog_manager-test.cc | 23 +++++++++++++++++++++++
src/kudu/master/catalog_manager.cc | 23 +++++------------------
src/kudu/master/master-test.cc | 1 -
src/kudu/master/ts_descriptor.cc | 27 ++++++++++++++++++++++++++-
src/kudu/master/ts_descriptor.h | 24 +++++++++++-------------
5 files changed, 65 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0b5545bc/src/kudu/master/catalog_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager-test.cc b/src/kudu/master/catalog_manager-test.cc
index b3dd302..bd48309 100644
--- a/src/kudu/master/catalog_manager-test.cc
+++ b/src/kudu/master/catalog_manager-test.cc
@@ -18,6 +18,7 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/master/catalog_manager.h"
+#include "kudu/master/ts_descriptor.h"
#include "kudu/util/test_util.h"
namespace kudu {
@@ -83,5 +84,27 @@ TEST(TableInfoTest, TestAssignmentRanges) {
}
}
+TEST(TestTSDescriptor, TestReplicaCreationsDecay) {
+ TSDescriptor ts("test");
+ ASSERT_EQ(0, ts.RecentReplicaCreations());
+ ts.IncrementRecentReplicaCreations();
+
+ // The load should start at close to 1.0.
+ double val_a = ts.RecentReplicaCreations();
+ ASSERT_NEAR(1.0, val_a, 0.05);
+
+ // After 10ms it should have dropped a bit, but still be close to 1.0.
+ SleepFor(MonoDelta::FromMilliseconds(10));
+ double val_b = ts.RecentReplicaCreations();
+ ASSERT_LT(val_b, val_a);
+ ASSERT_NEAR(0.99, val_a, 0.05);
+
+ if (AllowSlowTests()) {
+ // After 10 seconds, we should have dropped to 0.5^(10/60) = 0.891
+ SleepFor(MonoDelta::FromSeconds(10));
+ ASSERT_NEAR(0.891, ts.RecentReplicaCreations(), 0.05);
+ }
+}
+
} // namespace master
} // namespace kudu
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0b5545bc/src/kudu/master/catalog_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index e1ecadb..9a9344a 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -346,19 +346,7 @@ void CatalogManagerBgTasks::Shutdown() {
}
void CatalogManagerBgTasks::Run() {
- MonoTime last_process_time = MonoTime::Now(MonoTime::FINE);
while (!NoBarrier_Load(&closing_)) {
- MonoTime now = MonoTime::Now(MonoTime::FINE);
- double since_last_process = now.GetDeltaSince(last_process_time).ToSeconds();
- last_process_time = now;
-
- // Decay load estimates on tablet servers.
- TSDescriptorVector ts_descs;
- catalog_manager_->master_->ts_manager()->GetAllLiveDescriptors(&ts_descs);
- for (const auto& ts : ts_descs) {
- ts->DecayRecentReplicaCreations(since_last_process);
- }
-
// Perform assignment processing.
if (!catalog_manager_->IsInitialized()) {
LOG(WARNING) << "Catalog manager is not initialized!";
@@ -2852,8 +2840,8 @@ shared_ptr<TSDescriptor> CatalogManager::PickBetterReplicaLocation(
//
// TODO: in the future we may want to factor in other items such as available disk space,
// actual request load, etc.
- double load_a = a->recent_replica_creations() + a->num_live_replicas();
- double load_b = b->recent_replica_creations() + b->num_live_replicas();
+ double load_a = a->RecentReplicaCreations() + a->num_live_replicas();
+ double load_b = b->RecentReplicaCreations() + b->num_live_replicas();
if (load_a < load_b) {
return a;
} else if (load_b < load_a) {
@@ -2922,10 +2910,9 @@ void CatalogManager::SelectReplicas(const TSDescriptorVector& ts_descs,
InsertOrDie(&already_selected, ts);
// Increment the number of pending replicas so that we take this selection into
- // account when assigning replicas for other tablets of the same table.
- //
- // This value gets decayed by the catalog manager background task.
- ts->increment_recent_replica_creations();
+ // account when assigning replicas for other tablets of the same table. This
+ // value decays back to 0 over time.
+ ts->IncrementRecentReplicaCreations();
TSRegistrationPB reg;
ts->GetRegistration(®);
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0b5545bc/src/kudu/master/master-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 9abf32f..9fb52db 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -271,7 +271,6 @@ TEST_F(MasterTest, TestCatalog) {
ASSERT_EQ(1, tables.tables_size());
ASSERT_EQ(kTableName, tables.tables(0).name());
-
// Delete the table
{
DeleteTableRequestPB req;
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0b5545bc/src/kudu/master/ts_descriptor.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/ts_descriptor.cc b/src/kudu/master/ts_descriptor.cc
index 8fa477b..d92a600 100644
--- a/src/kudu/master/ts_descriptor.cc
+++ b/src/kudu/master/ts_descriptor.cc
@@ -49,6 +49,7 @@ TSDescriptor::TSDescriptor(std::string perm_id)
last_heartbeat_(MonoTime::Now(MonoTime::FINE)),
has_tablet_report_(false),
recent_replica_creations_(0),
+ last_replica_creations_decay_(MonoTime::Now(MonoTime::FINE)),
num_live_replicas_(0) {
}
@@ -110,9 +111,33 @@ void TSDescriptor::set_has_tablet_report(bool has_report) {
has_tablet_report_ = has_report;
}
-void TSDescriptor::DecayRecentReplicaCreations(double secs_since_last_decay) {
+void TSDescriptor::DecayRecentReplicaCreationsUnlocked() {
+ // In most cases, we won't have any recent replica creations, so
+ // we don't need to bother calling the clock, etc.
+ if (recent_replica_creations_ == 0) return;
+
const double kHalflifeSecs = 60;
+ MonoTime now = MonoTime::Now(MonoTime::FINE);
+ double secs_since_last_decay = now.GetDeltaSince(last_replica_creations_decay_).ToSeconds();
recent_replica_creations_ *= pow(0.5, secs_since_last_decay / kHalflifeSecs);
+
+ // If sufficiently small, reset down to 0 to take advantage of the fast path above.
+ if (recent_replica_creations_ < 1e-12) {
+ recent_replica_creations_ = 0;
+ }
+ last_replica_creations_decay_ = now;
+}
+
+void TSDescriptor::IncrementRecentReplicaCreations() {
+ lock_guard<simple_spinlock> l(&lock_);
+ DecayRecentReplicaCreationsUnlocked();
+ recent_replica_creations_ += 1;
+}
+
+double TSDescriptor::RecentReplicaCreations() {
+ boost::lock_guard<simple_spinlock> l(lock_);
+ DecayRecentReplicaCreationsUnlocked();
+ return recent_replica_creations_;
}
void TSDescriptor::GetRegistration(TSRegistrationPB* reg) const {
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/0b5545bc/src/kudu/master/ts_descriptor.h
----------------------------------------------------------------------
diff --git a/src/kudu/master/ts_descriptor.h b/src/kudu/master/ts_descriptor.h
index 404e8f4..b0327fc 100644
--- a/src/kudu/master/ts_descriptor.h
+++ b/src/kudu/master/ts_descriptor.h
@@ -90,22 +90,15 @@ class TSDescriptor {
Status GetConsensusProxy(const std::shared_ptr<rpc::Messenger>& messenger,
std::shared_ptr<consensus::ConsensusServiceProxy>* proxy);
- void increment_recent_replica_creations() {
- lock_guard<simple_spinlock> l(&lock_);
- recent_replica_creations_ += 1;
- }
-
- // Decay the accounting of how many replicas have been recently
- // created on this host.
- void DecayRecentReplicaCreations(double secs_since_last_decay);
+ // Increment the accounting of the number of replicas recently created on this
+ // server. This value will automatically decay over time.
+ void IncrementRecentReplicaCreations();
// Return the number of replicas which have recently been created on this
// TS. This number is incremented when replicas are placed on the TS, and
- // then decayed over time.
- double recent_replica_creations() const {
- lock_guard<simple_spinlock> l(&lock_);
- return recent_replica_creations_;
- }
+ // then decayed over time. This method is not 'const' because each call
+ // actually performs the time-based decay.
+ double RecentReplicaCreations();
// Set the number of live replicas (i.e. running or bootstrapping).
void set_num_live_replicas(int n) {
@@ -121,11 +114,15 @@ class TSDescriptor {
}
private:
+ FRIEND_TEST(TestTSDescriptor, TestReplicaCreationsDecay);
+
explicit TSDescriptor(std::string perm_id);
// Uses DNS to resolve registered hosts to a single Sockaddr.
Status ResolveSockaddr(Sockaddr* addr) const;
+ void DecayRecentReplicaCreationsUnlocked();
+
mutable simple_spinlock lock_;
const std::string permanent_uuid_;
@@ -140,6 +137,7 @@ class TSDescriptor {
// The number of times this tablet server has recently been selected to create a
// tablet replica. This value decays back to 0 over time.
double recent_replica_creations_;
+ MonoTime last_replica_creations_decay_;
// The number of live replicas on this host, from the last heartbeat.
int num_live_replicas_;