You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by da...@apache.org on 2017/02/23 01:37:12 UTC

[1/2] kudu git commit: [docs] added info on TSK an CA entries in systable

Repository: kudu
Updated Branches:
  refs/heads/master 914ed3b98 -> 69f2619d0


[docs] added info on TSK an CA entries in systable

Change-Id: I3c75d792981c3152f5844a2ea6934c0ce0032522
Reviewed-on: http://gerrit.cloudera.org:8080/6111
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0bd10cb8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0bd10cb8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0bd10cb8

Branch: refs/heads/master
Commit: 0bd10cb881d389eded337eb188a0bf30cc2483f3
Parents: 914ed3b
Author: Alexey Serbin <as...@cloudera.com>
Authored: Wed Feb 22 12:09:03 2017 -0800
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Thu Feb 23 01:27:32 2017 +0000

----------------------------------------------------------------------
 docs/design-docs/master.md | 32 +++++++++++++++++++++++++++++---
 1 file changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/0bd10cb8/docs/design-docs/master.md
----------------------------------------------------------------------
diff --git a/docs/design-docs/master.md b/docs/design-docs/master.md
index 4a366aa..fdcf7dd 100644
--- a/docs/design-docs/master.md
+++ b/docs/design-docs/master.md
@@ -104,9 +104,6 @@ Master or tablet server goes offline before a particular DeleteTablet operation
 successfully completes, the Master will send a new DeleteTablet request at the
 time that the next heartbeat is received from the tablet that is to be deleted.
 
-A "Cleaner" process will be reponsible for removing the data from deleted tables
-and tablets in the future, both on-disk and cached in memory (TODO).
-
 
 Table Assignment (Tablet Creation)
 ----------------------------------
@@ -237,3 +234,32 @@ tablet server to the master (tablet servers that have been heard from,
 heartbeats, tablet reports, etc...). The information is stored in a
 map, where the key is the permanent uuid of a tablet server and the
 value is (a pointer to) a TSDescriptor.
+
+IPKI: Internal Root Certificate Authority (CA) Information
+----------------------------------------------------------
+
+Besides tables' metadata, the system table contains the root CA certificate
+and corresponding private key when Kudu is configured to use its own IPKI
+(Internal Private Key Infrastructure). The root CA certificate and the private
+key are used to
+  1. Sign TLS certificates for Kudu server-side components like Master and
+     Tablet Servers.
+  2. Authenticate the server side of TLS connection: the initiator of a
+     TLS connection (the client side) uses Kudu CA certificate to make sure
+     the peer has valid TLS certificate signed by the Kudu internal CA.
+
+Upon start of a Kudu master server, it generates and stores the root CA
+certificate and corresponding private key when becoming leader if no such
+information is present in the system table. If the internal root CA information
+is already present in the system table, the leader master loads that
+information into memory and uses it appropriately.
+
+IPKI: TSK (Token Signing Keys)
+------------------------------
+
+The system table contains entries with TSKs used for authn/authz token signing.
+The leader master generates and stores those in the system table. Upon start-up
+or on the change of master leadership, a new leader master loads existing TSK
+entries from the system table and populates in-memory structures necessary
+for token signing. Expired keys are lazily purged from the system table
+by the leader master.


[2/2] kudu git commit: KUDU-1493: Implement SchemaRelationProvider

Posted by da...@apache.org.
KUDU-1493: Implement SchemaRelationProvider

Implement SchemaRelationProvider org.apache.kudu.spark.kudu.DefaultSource to allow
for specifying a user schema on read and thus allow for DataFrames of different
column orderings

Change-Id: I8b6073256b61a174f898be222058277be976273c
Reviewed-on: http://gerrit.cloudera.org:8080/5167
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/69f2619d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/69f2619d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/69f2619d

Branch: refs/heads/master
Commit: 69f2619d059e6aeb45f947b8367069b12f3b1bb9
Parents: 0bd10cb
Author: Drew Manlove <an...@codefutures.com>
Authored: Mon Nov 21 14:14:06 2016 -0700
Committer: Dan Burkert <da...@apache.org>
Committed: Thu Feb 23 01:36:37 2017 +0000

----------------------------------------------------------------------
 .../apache/kudu/spark/kudu/DefaultSource.scala  | 71 +++++++++++++-------
 .../kudu/spark/kudu/DefaultSourceTest.scala     | 42 ++++++++++--
 2 files changed, 85 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/69f2619d/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
index af1d573..5a6c178 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/DefaultSource.scala
@@ -24,7 +24,8 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
-import org.apache.kudu.Type
+import org.apache.kudu.{Type, ColumnSchema}
+
 import org.apache.kudu.annotations.InterfaceStability
 import org.apache.kudu.client.KuduPredicate.ComparisonOp
 import org.apache.kudu.client._
@@ -38,7 +39,8 @@ import org.apache.kudu.client._
   * operations through [[org.apache.spark.sql.DataFrameReader.format]].
   */
 @InterfaceStability.Unstable
-class DefaultSource extends RelationProvider with CreatableRelationProvider {
+class DefaultSource extends RelationProvider with CreatableRelationProvider
+  with SchemaRelationProvider {
 
   val TABLE_KEY = "kudu.table"
   val KUDU_MASTER = "kudu.master"
@@ -58,18 +60,9 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
       throw new IllegalArgumentException(
         s"Kudu table name must be specified in create options using key '$TABLE_KEY'"))
     val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+    val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
 
-    val opParam = parameters.getOrElse(OPERATION, "upsert")
-    val operationType = opParam.toLowerCase match {
-      case "insert" => Insert
-      case "insert-ignore" => InsertIgnore
-      case "upsert" => Upsert
-      case "update" => Update
-      case "delete" => Delete
-      case _ => throw new IllegalArgumentException(s"Unsupported operation type '$opParam'")
-    }
-
-    new KuduRelation(tableName, kuduMaster, operationType)(sqlContext)
+    new KuduRelation(tableName, kuduMaster, operationType, None)(sqlContext)
   }
 
   /**
@@ -93,6 +86,28 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
 
     kuduRelation
   }
+
+  override def createRelation(sqlContext: SQLContext, parameters: Map[String, String],
+                              schema: StructType): BaseRelation = {
+    val tableName = parameters.getOrElse(TABLE_KEY,
+      throw new IllegalArgumentException(s"Kudu table name must be specified in create options " +
+        s"using key '$TABLE_KEY'"))
+    val kuduMaster = parameters.getOrElse(KUDU_MASTER, "localhost")
+    val operationType = getOperationType(parameters.getOrElse(OPERATION, "upsert"))
+
+    new KuduRelation(tableName, kuduMaster, operationType, Some(schema))(sqlContext)
+  }
+
+  private def getOperationType(opParam: String): OperationType = {
+    opParam.toLowerCase match {
+      case "insert" => Insert
+      case "insert-ignore" => InsertIgnore
+      case "upsert" => Upsert
+      case "update" => Update
+      case "delete" => Delete
+      case _ => throw new IllegalArgumentException(s"Unsupported operation type '$opParam'")
+    }
+  }
 }
 
 /**
@@ -101,16 +116,18 @@ class DefaultSource extends RelationProvider with CreatableRelationProvider {
   * @param tableName Kudu table that we plan to read from
   * @param masterAddrs Kudu master addresses
   * @param operationType The default operation type to perform when writing to the relation
+  * @param userSchema A schema used to select columns for the relation
   * @param sqlContext SparkSQL context
   */
 @InterfaceStability.Unstable
 class KuduRelation(private val tableName: String,
                    private val masterAddrs: String,
-                   private val operationType: OperationType)(
-                   val sqlContext: SQLContext)
-extends BaseRelation
-with PrunedFilteredScan
-with InsertableRelation {
+                   private val operationType: OperationType,
+                   private val userSchema: Option[StructType])(
+                    val sqlContext: SQLContext)
+  extends BaseRelation
+    with PrunedFilteredScan
+    with InsertableRelation {
 
   import KuduRelation._
 
@@ -127,13 +144,19 @@ with InsertableRelation {
     * @return schema generated from the Kudu table's schema
     */
   override def schema: StructType = {
-    val fields: Array[StructField] =
-      table.getSchema.getColumns.asScala.map { columnSchema =>
-        val sparkType = kuduTypeToSparkType(columnSchema.getType)
-        StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
-      }.toArray
+    userSchema match {
+      case Some(x) =>
+        StructType(x.fields.map(uf => table.getSchema.getColumn(uf.name))
+          .map(kuduColumnToSparkField))
+      case None =>
+        StructType(table.getSchema.getColumns.asScala.map(kuduColumnToSparkField).toArray)
+    }
+  }
 
-    new StructType(fields)
+  def kuduColumnToSparkField: (ColumnSchema) => StructField = {
+    columnSchema =>
+      val sparkType = kuduTypeToSparkType(columnSchema.getType)
+      new StructField(columnSchema.getName, sparkType, columnSchema.isNullable)
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/kudu/blob/69f2619d/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
index 3dbd088..9790e81 100644
--- a/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/apache/kudu/spark/kudu/DefaultSourceTest.scala
@@ -25,19 +25,21 @@ import scala.collection.immutable.IndexedSeq
 import scala.util.control.NonFatal
 
 import com.google.common.collect.ImmutableList
-import org.apache.spark.sql.SQLContext
+
 import org.apache.spark.sql.functions._
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.types.{DataTypes, StructField, StructType};
 import org.junit.Assert._
 import org.junit.runner.RunWith
+import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
 import org.scalatest.junit.JUnitRunner
-import org.scalatest.{BeforeAndAfter, FunSuite}
 
 import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder
 import org.apache.kudu.{Schema, Type}
-import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.CreateTableOptions
 
 @RunWith(classOf[JUnitRunner])
-class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
+class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter with Matchers {
 
   test("timestamp conversion") {
     val epoch = new Timestamp(0)
@@ -450,4 +452,36 @@ class DefaultSourceTest extends FunSuite with TestContext with BeforeAndAfter {
     assertTrue(kuduContext.tableExists(newTable))
     assert(checkDf.count == 10)
   }
+
+  test("create relation with schema") {
+
+    // user-supplied schema that is compatible with actual schema, but with the key at the end
+    val userSchema: StructType = StructType(List(
+      StructField("c4_long", DataTypes.LongType),
+      StructField("key", DataTypes.IntegerType)
+    ))
+
+    val dfDefaultSchema = sqlContext.read.options(kuduOptions).kudu
+    assertEquals(11, dfDefaultSchema.schema.fields.length)
+
+    val dfWithUserSchema = sqlContext.read.options(kuduOptions).schema(userSchema).kudu
+    assertEquals(2, dfWithUserSchema.schema.fields.length)
+
+    dfWithUserSchema.limit(10).collect()
+    assertTrue(dfWithUserSchema.columns.deep == Array("c4_long", "key").deep)
+  }
+
+  test("create relation with invalid schema") {
+
+    // user-supplied schema that is NOT compatible with actual schema
+    val userSchema: StructType = StructType(List(
+      StructField("foo", DataTypes.LongType),
+      StructField("bar", DataTypes.IntegerType)
+    ))
+
+    intercept[IllegalArgumentException] {
+      sqlContext.read.options(kuduOptions).schema(userSchema).kudu
+    }.getMessage should include ("Unknown column: foo")
+
+  }
 }