You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2017/09/20 21:26:44 UTC
[2/2] kudu git commit: kudu-spark: change default master address to
local fqdn
kudu-spark: change default master address to local fqdn
The previous default was 'localhost', which isn't resolvable in a
cluster context. It would also cause issues on Kerberized clusters due
to 'localhost' not matching the Kerberos principal name of the master
(see KUDU-2142).
Change-Id: I10ec7414c451f54b95d86663d743162688e304ba
Reviewed-on: http://gerrit.cloudera.org:8080/8072
Reviewed-by: Attila Bukor <ab...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5a0015e7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5a0015e7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5a0015e7
Branch: refs/heads/master
Commit: 5a0015e75c6bde8ddb6f113083c3e7d46e48c5de
Parents: 3ab3f45
Author: Dan Burkert <da...@apache.org>
Authored: Thu Sep 14 11:43:38 2017 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Wed Sep 20 21:25:57 2017 +0000
----------------------------------------------------------------------
.../apache/kudu/spark/tools/ImportExportFiles.scala | 10 +++++++---
.../spark/tools/IntegrationTestBigLinkedList.scala | 16 ++++++++++------
.../org/apache/kudu/spark/kudu/DefaultSource.scala | 10 ++++++----
3 files changed, 23 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
index 025b06b..a09b52d 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/ImportExportFiles.scala
@@ -17,9 +17,11 @@
package org.apache.kudu.spark.tools
+import java.net.InetAddress
+
import org.apache.kudu.client.KuduClient
import org.apache.kudu.spark.tools.ImportExportKudu.ArgsCls
-import org.apache.spark.sql.{SQLContext, SparkSession}
+import org.apache.spark.sql.SparkSession
import org.apache.spark.SparkConf
import org.slf4j.{Logger, LoggerFactory}
import org.apache.kudu.spark.kudu._
@@ -33,13 +35,15 @@ object ImportExportKudu {
sys.exit(1)
}
+ def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
def usage: String =
s"""
| Usage: --operation=import/export --format=<data-format(csv,parquet,avro)> --master-addrs=<master-addrs> --path=<path> --table-name=<table-name>
| where
| operation: import or export data from or to Kudu tables, default: import
| format: specify the format of data want to import/export, the following formats are supported csv,parquet,avro default:csv
- | masterAddrs: comma separated addresses of Kudu master nodes, default: localhost
+ | masterAddrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
| path: path to input or output for import/export operation, default: file://
| tableName: table name to import/export, default: ""
| columns: columns name for select statement on export from kudu table, default: *
@@ -49,7 +53,7 @@ object ImportExportKudu {
case class ArgsCls(operation: String = "import",
format: String = "csv",
- masterAddrs: String = "localhost",
+ masterAddrs: String = defaultMasterAddrs,
path: String = "file://",
tableName: String = "",
columns: String = "*",
http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
index db03c24..7b1fc31 100644
--- a/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
+++ b/java/kudu-spark-tools/src/main/scala/org/apache/kudu/spark/tools/IntegrationTestBigLinkedList.scala
@@ -17,6 +17,8 @@
package org.apache.kudu.spark.tools
+import java.net.InetAddress
+
import com.google.common.annotations.VisibleForTesting
import org.apache.kudu.client.SessionConfiguration.FlushMode
import org.apache.kudu.client.{KuduClient, KuduSession, KuduTable}
@@ -81,6 +83,8 @@ object IntegrationTestBigLinkedList {
else s"${n}ns"
}
+ def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
def main(args: Array[String]): Unit = {
if (args.isEmpty) { fail(usage) }
@@ -94,7 +98,7 @@ object IntegrationTestBigLinkedList {
}
object Generator {
- import IntegrationTestBigLinkedList.{LOG, fail, nanosToHuman, parseIntFlag}
+ import IntegrationTestBigLinkedList.{LOG, defaultMasterAddrs, fail, nanosToHuman, parseIntFlag}
def usage: String =
s"""
@@ -108,7 +112,7 @@ object Generator {
| hashPartitions: number of hash partitions to create for the new linked list table, if it doesn't exist, default: 1
| rangePartitions: number of range partitions to create for the new linked list table, if it doesn't exist, default: 1
| replicas: number of replicas to create for the new linked list table, if it doesn't exist, default: 1
- | master-addrs: comma separated addresses of Kudu master nodes, default: localhost
+ | master-addrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
| table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
""".stripMargin
@@ -118,7 +122,7 @@ object Generator {
hashPartitions: Int = 1,
rangePartitions: Int = 1,
replicas: Int = 1,
- masterAddrs: String = "localhost",
+ masterAddrs: String = defaultMasterAddrs,
tableName: String = DEFAULT_TABLE_NAME)
object Args {
@@ -267,19 +271,19 @@ object Generator {
}
object Verifier {
- import IntegrationTestBigLinkedList.{fail, parseLongFlag}
+ import IntegrationTestBigLinkedList.{defaultMasterAddrs, fail, parseLongFlag}
def usage: String =
s"""
| Usage: verify --nodes=<nodes> --master-addrs=<master-addrs> --table-name=<table-name>
| where
| nodes: number of nodes expected to be in the linked list table
- | master-addrs: comma separated addresses of Kudu master nodes, default: localhost
+ | master-addrs: comma separated addresses of Kudu master nodes, default: $defaultMasterAddrs
| table-name: the name of the linked list table, default: $DEFAULT_TABLE_NAME
""".stripMargin
case class Args(nodes: Option[Long] = None,
- masterAddrs: String = "localhost",
+ masterAddrs: String = defaultMasterAddrs,
tableName: String = DEFAULT_TABLE_NAME)
object Args {
http://git-wip-us.apache.org/repos/asf/kudu/blob/5a0015e7/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index 5c91a1f..fbc2d38 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -17,6 +17,7 @@
package org.apache.kudu.spark.kudu
+import java.net.InetAddress
import java.sql.Timestamp
import scala.collection.JavaConverters._
@@ -49,6 +50,8 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
val OPERATION = "kudu.operation"
val FAULT_TOLERANT_SCANNER = "kudu.faultTolerantScan"
+ def defaultMasterAddrs: String = InetAddress.getLocalHost.getCanonicalHostName
+
/**
* Construct a BaseRelation using the provided context and parameters.
*
@@ -62,13 +65,12 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
val tableName = parameters.getOrElse(TABLE_KEY,
throw new IllegalArgumentException(
s"Kudu table name must be specified in create options using key '$TABLE_KEY'"))
- val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+ val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
.getOrElse(false)
- new KuduRelation(tableName, kuduMaster, faultTolerantScanner, operationType,
- None)(sqlContext)
+ new KuduRelation(tableName, kuduMaster, faultTolerantScanner, operationType, None)(sqlContext)
}
/**
@@ -97,7 +99,7 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider
val tableName = parameters.getOrElse(TABLE_KEY,
throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
s"using key '$TABLE_KEY'"))
- val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+ val kuduMaster = parameters.getOrElse(KUDU_MASTER, defaultMasterAddrs)
val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
val faultTolerantScanner = Try(parameters.getOrElse(FAULT_TOLERANT_SCANNER, "false").toBoolean)
.getOrElse(false)