You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2017/02/23 01:37:12 UTC
[1/2] kudu git commit: [docs] added info on TSK an CA entries in
systable
Repository: kudu
Updated Branches:
refs/heads/master 914ed3b98 -> 69f2619d0
[docs] added info on TSK an CA entries in systable
Change-Id: I3c75d792981c3152f5844a2ea6934c0ce0032522
Reviewed-on: http://gerrit.cloudera.org:8080/6111
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0bd10cb8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0bd10cb8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0bd10cb8
Branch: refs/heads/master
Commit: 0bd10cb881d389eded337eb188a0bf30cc2483f3
Parents: 914ed3b
Author: Alexey Serbin <as...@cloudera.com>
Authored: Wed Feb 22 12:09:03 2017 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Thu Feb 23 01:27:32 2017 +0000
----------------------------------------------------------------------
docs/design-docs/master.md | 32 +++++++++++++++++++++++++++++---
1 file changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/0bd10cb8/docs/design-docs/master.md
----------------------------------------------------------------------
diff --git a/docs/design-docs/master.md b/docs/design-docs/master.md
index 4a366aa..fdcf7dd 100644
--- a/docs/design-docs/master.md
+++ b/docs/design-docs/master.md
@@ -104,9 +104,6 @@ Master or tablet server goes offline before a particular DeleteTablet operation
successfully completes, the Master will send a new DeleteTablet request at the
time that the next heartbeat is received from the tablet that is to be deleted.
-A "Cleaner" process will be reponsible for removing the data from deleted tables
-and tablets in the future, both on-disk and cached in memory (TODO).
-
Table Assignment (Tablet Creation)
----------------------------------
@@ -237,3 +234,32 @@ tablet server to the master (tablet servers that have been heard from,
heartbeats, tablet reports, etc...). The information is stored in a
map, where the key is the permanent uuid of a tablet server and the
value is (a pointer to) a TSDescriptor.
+
+IPKI: Internal Root Certificate Authority (CA) Information
+----------------------------------------------------------
+
+Besides tables' metadata, the system table contains the root CA certificate
+and corresponding private key when Kudu is configured to use its own IPKI
+(Internal Private Key Infrastructure). The root CA certificate and the private
+key are used to
+ 1. Sign TLS certificates for Kudu server-side components like Master and
+ Tablet Servers.
+ 2. Authenticate the server side of TLS connection: the initiator of a
+ TLS connection (the client side) uses Kudu CA certificate to make sure
+ the peer has valid TLS certificate signed by the Kudu internal CA.
+
+Upon start of a Kudu master server, it generates and stores the root CA
+certificate and corresponding private key when becoming leader if no such
+information is present in the system table. If the internal root CA information
+is already present in the system table, the leader master loads that
+information into memory and uses it appropriately.
+
+IPKI: TSK (Token Signing Keys)
+------------------------------
+
+The system table contains entries with TSKs used for authn/authz token signing.
+The leader master generates and stores those in the system table. Upon start-up
+or on the change of master leadership, a new leader master loads existing TSK
+entries from the system table and populates in-memory structures necessary
+for token signing. Expired keys are lazily purged from the system table
+by the leader master.
[2/2] kudu git commit: KUDU-1493: Implement SchemaRelationProvider
Posted by da...@apache.org.
KUDU-1493: Implement SchemaRelationProvider
Implement SchemaRelationProvider org.apache.kudu.spark.kudu.DefaultSource to allow
for specifying a user schema on read and thus allow for DataFrames of different
column orderings
Change-Id: I8b6073256b61a174f898be222058277be976273c
Reviewed-on: http://gerrit.cloudera.org:8080/5167
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/69f2619d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/69f2619d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/69f2619d
Branch: refs/heads/master
Commit: 69f2619d059e6aeb45f947b8367069b12f3b1bb9
Parents: 0bd10cb
Author: Drew Manlove <an...@codefutures.com>
Authored: Mon Nov 21 14:14:06 2016 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Thu Feb 23 01:36:37 2017 +0000
----------------------------------------------------------------------
.../apache/kudu/spark/kudu/DefaultSource.scala | 71 +++++++++++++-------
.../kudu/spark/kudu/DefaultSourceTest.scala | 42 ++++++++++--
2 files changed, 85 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/69f2619d/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index af1d573..5a6c178 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -24,7 +24,8 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.apache.kudu.Type
+import org.apache.kudu.{Type, ColumnSchema}
+
import org.apache.kudu.annotations.InterfaceStability
import org.apache.kudu.client.KuduPredicate.ComparisonOp
import org.apache.kudu.client._
@@ -38,7 +39,8 @@ import org.apache.kudu.client._
* operations through [[org.apache.spark.sql.DataFrameReader.format]].
*/
@InterfaceStability.Unstable
-class DefaultSource extends RelationProvider with CreatableRelationProvider {
+class DefaultSource extends RelationProvider with CreatableRelationProvider
+ with SchemaRelationProvider {
val TABLE_KEY = "kudu.table"
val KUDU_MASTER = "kudu.master"
@@ -58,18 +60,9 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
throw new IllegalArgumentException(
s"Kudu table name must be specified in create options using key '$TABLE_KEY'"))
val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+ val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
- val opParam = parameters.getOrElse(OPERATION, "upsert")
- val operationType = opParam.toLowerCase match {
- case "insert" => Insert
- case "insert-ignore" => InsertIgnore
- case "upsert" => Upsert
- case "update" => Update
- case "delete" => Delete
- case _ => throw new IllegalArgumentException(s"Unsupported operation type '$opParam'")
- }
-
- new KuduRelation(tableName, kuduMaster, operationType)(sqlContext)
+ new KuduRelation(tableName, kuduMaster, operationType, None)(sqlContext)
}
/**
@@ -93,6 +86,28 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
kuduRelation
}
+
+ override def createRelation(sqlContext: SQLContext, parameters: Map[String, String],
+ schema: StructType): BaseRelation = {
+ val tableName = parameters.getOrElse(TABLE_KEY,
+ throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
+ s"using key '$TABLE_KEY'"))
+ val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+ val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
+
+ new KuduRelation(tableName, kuduMaster, operationType, Some(schema))(sqlContext)
+ }
+
+ private def getOperationType(opParam: String): OperationType = {
+ opParam.toLowerCase match {
+ case "insert" => Insert
+ case "insert-ignore" => InsertIgnore
+ case "upsert" => Upsert
+ case "update" => Update
+ case "delete" => Delete
+ case _ => throw new IllegalArgumentException(s"Unsupported operation type '$opParam'")
+ }
+ }
}
/**
@@ -101,16 +116,18 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
* @param tableName Kudu table that we plan to read from
* @param masterAddrs Kudu master addresses
* @param operationType The default operation type to perform when writing to the relation
+ * @param userSchema A schema used to select columns for the relation
* @param sqlContext SparkSQL context
*/
@InterfaceStability.Unstable
class KuduRelation(private val tableName: String,
private val masterAddrs: String,
- private val operationType: OperationType)(
- val sqlContext: SQLContext)
-extends BaseRelation
-with PrunedFilteredScan
-with InsertableRelation {
+ private val operationType: OperationType,
+ private val userSchema: Option[StructType])(
+ val sqlContext: SQLContext)
+ extends BaseRelation
+ with PrunedFilteredScan
+ with InsertableRelation {
import KuduRelation._
@@ -127,13 +144,19 @@ with InsertableRelation {
* @return schema generated from the Kudu table's schema
*/
override def schema: StructType = {
- val fields: Array[StructField] =
- table.getSchema.getColumns.asScala.map { columnSchema =>
- val sparkType = kuduTypeToSparkType(columnSchema.getType)
- StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
- }.toArray
+ userSchema match {
+ case Some(x) =>
+ StructType(x.fields.map(uf => table.getSchema.getColumn(uf.name))
+ .map(kuduColumnToSparkField))
+ case None =>
+ StructType(table.getSchema.getColumns.asScala.map(kuduColumnToSparkField).toArray)
+ }
+ }
- new StructType(fields)
+ def kuduColumnToSparkField: (ColumnSchema) => StructField = {
+ columnSchema =>
+ val sparkType = kuduTypeToSparkType(columnSchema.getType)
+ new StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
}
/**
http://git-wip-us.apache.org/repos/asf/kudu/blob/69f2619d/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index 3dbd088..9790e81 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -25,19 +25,21 @@ import scala.collection.immutable.IndexedSeq
import scala.util.control.NonFatal
import com.google.common.collect.ImmutableList
-import org.apache.spark.sql.SQLContext
+
import org.apache.spark.sql.functions._
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.types.{DataTypes, StructField, StructType};
import org.junit.Assert._
import org.junit.runner.RunWith
+import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
import org.scalatest.junit.JUnitRunner
-import org.scalatest.{BeforeAndAfter, FunSuite}
import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
import org.apache.kudu.{Schema, Type}
-import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.CreateTableOptions
@RunWith(classOf[JUnitRunner])
-class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
+class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter with Matchers {
test("timestamp conversion") {
val epoch = new Timestamp(0)
@@ -450,4 +452,36 @@ class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
assertTrue(kuduContext.tableExists(newTable))
assert(checkDf.count == 10)
}
+
+ test("create relation with schema") {
+
+ // user-supplied schema that is compatible with actual schema, but with the key at the end
+ val userSchema: StructType = StructType(List(
+ StructField("c4_long", DataTypes.LongType),
+ StructField("key", DataTypes.IntegerType)
+ ))
+
+ val dfDefaultSchema = sqlContext.read.options(kuduOptions).kudu
+ assertEquals(11, dfDefaultSchema.schema.fields.length)
+
+ val dfWithUserSchema = sqlContext.read.options(kuduOptions).schema(userSchema).kudu
+ assertEquals(2, dfWithUserSchema.schema.fields.length)
+
+ dfWithUserSchema.limit(10).collect()
+ assertTrue(dfWithUserSchema.columns.deep == Array("c4_long", "key").deep)
+ }
+
+ test("create relation with invalid schema") {
+
+ // user-supplied schema that is NOT compatible with actual schema
+ val userSchema: StructType = StructType(List(
+ StructField("foo", DataTypes.LongType),
+ StructField("bar", DataTypes.IntegerType)
+ ))
+
+ intercept[IllegalArgumentException] {
+ sqlContext.read.options(kuduOptions).schema(userSchema).kudu
+ }.getMessage should include ("Unknown column: foo")
+
+ }
}