You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2018/02/17 01:47:31 UTC
[1/3] kudu git commit: [rpc] fix typo in Messenger::metric_entity()
Repository: kudu
Updated Branches:
refs/heads/master 3bd1aad6d -> 9c70e0751
[rpc] fix typo in Messenger::metric_entity()
Change-Id: Ic976a5038e982d92318ca5441866fbd05858b92a
Reviewed-on: http://gerrit.cloudera.org:8080/9351
Tested-by: Alexey Serbin <as...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc2e8787
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc2e8787
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc2e8787
Branch: refs/heads/master
Commit: fc2e8787008445e087c0684fc612ace1964e9c24
Parents: 3bd1aad
Author: Alexey Serbin <as...@cloudera.com>
Authored: Thu Feb 15 22:38:51 2018 -0800
Committer: Todd Lipcon <to...@apache.org>
Committed: Fri Feb 16 21:27:51 2018 +0000
----------------------------------------------------------------------
src/kudu/rpc/messenger.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/fc2e8787/src/kudu/rpc/messenger.h
----------------------------------------------------------------------
diff --git a/src/kudu/rpc/messenger.h b/src/kudu/rpc/messenger.h
index e030a5c..129ad0b 100644
--- a/src/kudu/rpc/messenger.h
+++ b/src/kudu/rpc/messenger.h
@@ -308,7 +308,7 @@ class Messenger {
return closing_;
}
- scoped_refptr<MetricEntity> metric_entity() const { return metric_entity_.get(); }
+ scoped_refptr<MetricEntity> metric_entity() const { return metric_entity_; }
const int64_t rpc_negotiation_timeout_ms() const { return rpc_negotiation_timeout_ms_; }
[3/3] kudu git commit: [kudu-jepsen] run kudu CLI tool under 'kudu'
user
Posted by al...@apache.org.
[kudu-jepsen] run kudu CLI tool under 'kudu' user
Since cd41990 was committed, the 'kudu cluster ksck' CLI tool command
started returning failure exit code if running not under the service
superuser account (i.e. not under 'kudu' user) because fetching Raft
consensus information on a tablet requires service superuser-level
privileges.
To fix that, at least the two alternatives are available:
* Run the CLI tool under the service superuser account ('kudu' user).
* Run the 'kudu cluster ksck' sub-command with the
--consensus=false option, so it would not try to fetch consensus
information on existing tablets.
The former alternative makes more sense because the list of privileges
required to call server methods is evolving. Also, in future we may
need to do other queries using the kudu CLI tool in the context of
Jepsen tests for Kudu.
Change-Id: I7e556e901308871d0bbe6091be41a7a365421222
Reviewed-on: http://gerrit.cloudera.org:8080/9356
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao <ha...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9c70e075
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9c70e075
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9c70e075
Branch: refs/heads/master
Commit: 9c70e0751395c0418f23e90200882eb312fee5e9
Parents: a185432
Author: Alexey Serbin <as...@cloudera.com>
Authored: Fri Feb 16 14:47:42 2018 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Sat Feb 17 01:46:36 2018 +0000
----------------------------------------------------------------------
java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/9c70e075/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
----------------------------------------------------------------------
diff --git a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
index 2e21794..2fa75a2 100644
--- a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
+++ b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/util.clj
@@ -124,7 +124,7 @@
This function should be called in the context of already established SSH
session at the node."
[test node]
- (try (c/exec (kudu-cli test node) :table :list node)
+ (try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :table :list node)
true
(catch RuntimeException _ false)))
@@ -134,7 +134,8 @@
[test node tservers-count]
(let [pattern (str "Fetched info from all "
(str tservers-count)" Tablet Servers")]
- (try (c/exec (kudu-cli test node) :cluster :ksck node | :grep pattern)
+ (try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :cluster :ksck node |
+ :grep pattern)
true
(catch RuntimeException _ false))))
@@ -144,7 +145,7 @@
This function should be called in the context of already established SSH
session at the node."
[test node]
- (try (c/exec (kudu-cli test node) :tserver :status node)
+ (try (c/exec :sudo :-u kudu-uname (kudu-cli test node) :tserver :status node)
true
(catch RuntimeException _ false)))
[2/3] kudu git commit: [spark] enable scan locality by default
Posted by al...@apache.org.
[spark] enable scan locality by default
Commit 3abca98c5 introduced support to take advantage of scan locality
in Spark integration, however this feature is not enabled by default.
This patch defaults scan locality to use closest replica. And if the
client configures to not to use the default, then only list leader
replica as the perferred location to schedule a task.
Change-Id: Ifc94e9ce52877a88f96a06599a8657da929b6126
Reviewed-on: http://gerrit.cloudera.org:8080/9163
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a1854325
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a1854325
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a1854325
Branch: refs/heads/master
Commit: a1854325b1c769bafcd20504b5f50c72195bc790
Parents: fc2e878
Author: hahao <ha...@cloudera.com>
Authored: Tue Jan 30 13:54:44 2018 -0800
Committer: Hao Hao <ha...@cloudera.com>
Committed: Fri Feb 16 23:26:33 2018 +0000
----------------------------------------------------------------------
.../org/apache/kudu/spark/kudu/DefaultSource.scala | 4 ++--
.../main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala | 12 ++++++++++--
2 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/a1854325/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 7987bf8..58beec7 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -71,7 +71,7 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
.getOrElse(false)
- val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "leader_only"))
+ val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
new KuduRelation(tableName, kuduMaster, faultTolerantScanner,
scanLocality, operationType, None)(sqlContext)
@@ -107,7 +107,7 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
.getOrElse(false)
- val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "leader_only"))
+ val scanLocality = getScanLocalityType(parameters.getOrElse(SCAN_LOCALITY, "closest_replica"))
new KuduRelation(tableName, kuduMaster, faultTolerantScanner,
scanLocality, operationType, Some(schema))(sqlContext)
http://git-wip-us.apache.org/repos/asf/kudu/blob/a1854325/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
index 5a4ed8b..ddfdf46 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduRDD.scala
@@ -62,8 +62,16 @@ class KuduRDD private[kudu] (val kuduContext: KuduContext,
val tokens = builder.build().asScala
tokens.zipWithIndex.map {
case (token, index) =>
- new KuduPartition(index, token.serialize(),
- token.getTablet.getReplicas.asScala.map(_.getRpcHost).toArray)
+ // Only list the leader replica as the preferred location if
+ // replica selection policy is leader only, to take advantage
+ // of scan locality.
+ var locations: Array[String] = null
+ if (scanLocality == ReplicaSelection.LEADER_ONLY) {
+ locations = Array(token.getTablet.getLeaderReplica.getRpcHost)
+ } else {
+ locations = token.getTablet.getReplicas.asScala.map(_.getRpcHost).toArray
+ }
+ new KuduPartition(index, token.serialize(), locations)
}.toArray
}